diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml index ab88930a3a..b55d6728d2 100644 --- a/.github/workflows/e2e-reusable.yml +++ b/.github/workflows/e2e-reusable.yml @@ -41,6 +41,11 @@ on: description: 'PR number to run tests for.' required: false type: number + node_view_version: + description: 'Node View version to run tests with.' + required: false + default: '1' + type: string secrets: CYPRESS_RECORD_KEY: description: 'Cypress record key.' @@ -160,6 +165,7 @@ jobs: spec: '${{ inputs.spec }}' env: NODE_OPTIONS: --dns-result-order=ipv4first + CYPRESS_NODE_VIEW_VERSION: ${{ inputs.node_view_version }} CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} E2E_TESTS: true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e7400adecb..2f63f61bc6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -27,6 +27,11 @@ on: description: 'URL to call after workflow is done.' required: false default: '' + node_view_version: + description: 'Node View version to run tests with.' + required: false + default: '1' + type: string jobs: calls-start-url: @@ -46,6 +51,7 @@ jobs: branch: ${{ github.event.inputs.branch || 'master' }} user: ${{ github.event.inputs.user || 'PR User' }} spec: ${{ github.event.inputs.spec || 'e2e/*' }} + node_view_version: ${{ github.event.inputs.node_view_version || '1' }} secrets: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 681de6c024..0c5abcba47 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,6 +7,7 @@ "EditorConfig.EditorConfig", "esbenp.prettier-vscode", "mjmlio.vscode-mjml", - "Vue.volar" + "Vue.volar", + "vitest.explorer" ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index ab14dc462e..baa7b95b7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,96 @@ +# [1.65.0](https://github.com/n8n-io/n8n/compare/n8n@1.64.0...n8n@1.65.0) (2024-10-24) + + +### Bug Fixes + +* **AI Agent Node:** Preserve `intermediateSteps` when using output parser with non-tool agent ([#11363](https://github.com/n8n-io/n8n/issues/11363)) ([e61a853](https://github.com/n8n-io/n8n/commit/e61a8535aa39653b9a87575ea911a65318282167)) +* **API:** `PUT /credentials/:id` should move the specified credential, not the first one in the database ([#11365](https://github.com/n8n-io/n8n/issues/11365)) ([e6b2f8e](https://github.com/n8n-io/n8n/commit/e6b2f8e7e6ebbb6e3776a976297d519e99ac6c64)) +* **API:** Correct credential schema for response in `POST /credentials` ([#11340](https://github.com/n8n-io/n8n/issues/11340)) ([f495875](https://github.com/n8n-io/n8n/commit/f4958756b4976e0b608b9155dab84564f7e8804e)) +* **core:** Account for waiting jobs during shutdown ([#11338](https://github.com/n8n-io/n8n/issues/11338)) ([c863abd](https://github.com/n8n-io/n8n/commit/c863abd08300b53ea898fc4d06aae97dec7afa9b)) +* **core:** Add missing primary key to execution annotation tags table ([#11168](https://github.com/n8n-io/n8n/issues/11168)) ([b4b543d](https://github.com/n8n-io/n8n/commit/b4b543d41daa07753eca24ab93bf7445f672361d)) +* **core:** Change dedupe value column type from varchar(255) to text ([#11357](https://github.com/n8n-io/n8n/issues/11357)) ([7a71cff](https://github.com/n8n-io/n8n/commit/7a71cff4d75fe4e7282a398b4843428e0161ba8c)) +* **core:** Do not debounce webhooks, triggers and pollers activation ([#11306](https://github.com/n8n-io/n8n/issues/11306)) ([64bddf8](https://github.com/n8n-io/n8n/commit/64bddf86536ddd688638a643d24f80c947a12f31)) +* **core:** Enforce nodejs version consistently ([#11323](https://github.com/n8n-io/n8n/issues/11323)) ([0fa2e8c](https://github.com/n8n-io/n8n/commit/0fa2e8ca85005362d9043d82469f3c3525f4c4ef)) +* **core:** Fix memory issue with empty model response ([#11300](https://github.com/n8n-io/n8n/issues/11300)) ([216b119](https://github.com/n8n-io/n8n/commit/216b119350949de70f15cf2d61f474770803ad7a)) +* **core:** Fix race condition when resolving post-execute promise ([#11360](https://github.com/n8n-io/n8n/issues/11360)) ([4f1816e](https://github.com/n8n-io/n8n/commit/4f1816e03db00219bc2e723e3048848aef7f8fe1)) +* **core:** Sanitise IdP provided information in SAML test pages ([#11171](https://github.com/n8n-io/n8n/issues/11171)) ([74fc388](https://github.com/n8n-io/n8n/commit/74fc3889b946e8f224e65ef8d3d44125404aa4fc)) +* Don't show pin button in input panel when there's binary data ([#11267](https://github.com/n8n-io/n8n/issues/11267)) ([c0b5b92](https://github.com/n8n-io/n8n/commit/c0b5b92f62a2d7ba60492eb27daced268b654fe9)) +* **editor:** Add Personal project to main navigation ([#11161](https://github.com/n8n-io/n8n/issues/11161)) ([1f441f9](https://github.com/n8n-io/n8n/commit/1f441f97528f58e905eaf8930577bbcd08debf06)) +* **editor:** Fix Cannot read properties of undefined (reading 'finished') ([#11367](https://github.com/n8n-io/n8n/issues/11367)) ([475d72e](https://github.com/n8n-io/n8n/commit/475d72e0bc9e13c6dc56129902f6f89c67547f78)) +* **editor:** Fix delete all existing executions ([#11352](https://github.com/n8n-io/n8n/issues/11352)) ([3ec103f](https://github.com/n8n-io/n8n/commit/3ec103f8baaa89e579844947d945f00bec9e498e)) +* **editor:** Fix pin data button disappearing after reload ([#11198](https://github.com/n8n-io/n8n/issues/11198)) ([3b2f63e](https://github.com/n8n-io/n8n/commit/3b2f63e248cd0cba04087e2f40e13d670073707d)) +* **editor:** Fix RunData non-binary pagination when binary data is present ([#11309](https://github.com/n8n-io/n8n/issues/11309)) ([901888d](https://github.com/n8n-io/n8n/commit/901888d5b1027098653540c72f787f176941f35a)) +* **editor:** Fix sorting problem in older browsers that don't support `toSorted` ([#11204](https://github.com/n8n-io/n8n/issues/11204)) ([c728a2f](https://github.com/n8n-io/n8n/commit/c728a2ffe01f510a237979a54897c4680a407800)) +* **editor:** Follow-up fixes to projects side menu ([#11327](https://github.com/n8n-io/n8n/issues/11327)) ([4dde772](https://github.com/n8n-io/n8n/commit/4dde772814c55e66efcc9b369ae443328af21b14)) +* **editor:** Keep always focus on the first item on the node's search panel ([#11193](https://github.com/n8n-io/n8n/issues/11193)) ([c57cac9](https://github.com/n8n-io/n8n/commit/c57cac9e4d447c3a4240a565f9f2de8aa3b7c513)) +* **editor:** Open Community+ enrollment modal only for the instance owner ([#11292](https://github.com/n8n-io/n8n/issues/11292)) ([76724c3](https://github.com/n8n-io/n8n/commit/76724c3be6e001792433045c2b2aac0ef16d4b8a)) +* **editor:** Record sessionStarted telemetry event in Setting Store ([#11334](https://github.com/n8n-io/n8n/issues/11334)) ([1b734dd](https://github.com/n8n-io/n8n/commit/1b734dd9f42885594ce02400cfb395a4f5e7e088)) +* Ensure NDV params don't get cut off early and scrolled to the top ([#11252](https://github.com/n8n-io/n8n/issues/11252)) ([054fe97](https://github.com/n8n-io/n8n/commit/054fe9745ff6864f9088aa4cd66ed9e7869520d5)) +* **HTTP Request Tool Node:** Fix the undefined response issue when authentication is enabled ([#11343](https://github.com/n8n-io/n8n/issues/11343)) ([094ec68](https://github.com/n8n-io/n8n/commit/094ec68d4c00848013aa4eec4ac5efbd2c92afc5)) +* Include error in the message in JS task runner sandbox ([#11359](https://github.com/n8n-io/n8n/issues/11359)) ([0708b3a](https://github.com/n8n-io/n8n/commit/0708b3a1f8097af829c92fe106ea6ba375d6c500)) +* **Microsoft SQL Node:** Fix execute query to allow for non select query to run ([#11335](https://github.com/n8n-io/n8n/issues/11335)) ([ba158b4](https://github.com/n8n-io/n8n/commit/ba158b4f8533bd3430db8766d4921f75db5c1a11)) +* **OpenAI Chat Model Node, Ollama Chat Model Node:** Change default model to a more up-to-date option ([#11293](https://github.com/n8n-io/n8n/issues/11293)) ([0be04c6](https://github.com/n8n-io/n8n/commit/0be04c6348d8c059a96c3d37a6d6cd587bfb97f3)) +* **Pinecone Vector Store Node:** Prevent populating of vectors after manually stopping the execution ([#11288](https://github.com/n8n-io/n8n/issues/11288)) ([fbae17d](https://github.com/n8n-io/n8n/commit/fbae17d8fb35a5197fa183e3639bb36762dc73d2)) +* **Postgres Node:** Special datetime values cause errors ([#11225](https://github.com/n8n-io/n8n/issues/11225)) ([3c57f46](https://github.com/n8n-io/n8n/commit/3c57f46aaeb968d2974f2dc9790317a6a6fab624)) +* Resend invite operation on users list ([#11351](https://github.com/n8n-io/n8n/issues/11351)) ([e4218de](https://github.com/n8n-io/n8n/commit/e4218debd18812fa3aa508339afd3de03c4d69dc)) +* **SSH Node:** Cleanup temporary binary files as soon as possible ([#11305](https://github.com/n8n-io/n8n/issues/11305)) ([08a7b5b](https://github.com/n8n-io/n8n/commit/08a7b5b7425663ec6593114921c2e22ab37d039e)) + + +### Features + +* Add report bug buttons ([#11304](https://github.com/n8n-io/n8n/issues/11304)) ([296f68f](https://github.com/n8n-io/n8n/commit/296f68f041b93fd32ac7be2b53c2b41d58c2998a)) +* **AI Agent Node:** Make tools optional when using OpenAI model with Tools agent ([#11212](https://github.com/n8n-io/n8n/issues/11212)) ([fed7c3e](https://github.com/n8n-io/n8n/commit/fed7c3ec1fb0553adaa9a933f91aabfd54fe83a3)) +* **core:** introduce JWT API keys for the public API ([#11005](https://github.com/n8n-io/n8n/issues/11005)) ([679fa4a](https://github.com/n8n-io/n8n/commit/679fa4a10a85fc96e12ca66fe12cdb32368bc12b)) +* **core:** Enforce config file permissions on startup ([#11328](https://github.com/n8n-io/n8n/issues/11328)) ([c078a51](https://github.com/n8n-io/n8n/commit/c078a516bec857831cc904ef807d0791b889f3a2)) +* **core:** Handle cycles in workflows when partially executing them ([#11187](https://github.com/n8n-io/n8n/issues/11187)) ([321d6de](https://github.com/n8n-io/n8n/commit/321d6deef18806d88d97afef2f2c6f29e739ccb4)) +* **editor:** Separate node output execution tooltip from status icon ([#11196](https://github.com/n8n-io/n8n/issues/11196)) ([cd15e95](https://github.com/n8n-io/n8n/commit/cd15e959c7af82a7d8c682e94add2b2640624a70)) +* **GitHub Node:** Add workflow resource operations ([#10744](https://github.com/n8n-io/n8n/issues/10744)) ([d309112](https://github.com/n8n-io/n8n/commit/d3091126472faa2c8f270650e54027d19dc56bb6)) +* **n8n Form Page Node:** New node ([#10390](https://github.com/n8n-io/n8n/issues/10390)) ([643d66c](https://github.com/n8n-io/n8n/commit/643d66c0ae084a0d93dac652703adc0a32cab8de)) +* **n8n Google My Business Node:** New node ([#10504](https://github.com/n8n-io/n8n/issues/10504)) ([bf28fbe](https://github.com/n8n-io/n8n/commit/bf28fbefe5e8ba648cba1555a2d396b75ee32bbb)) +* Run `mfa.beforeSetup` hook before enabling MFA ([#11116](https://github.com/n8n-io/n8n/issues/11116)) ([25c1c32](https://github.com/n8n-io/n8n/commit/25c1c3218cf1075ca3abd961236f3b2fbd9d6ba9)) +* **Structured Output Parser Node:** Refactor Output Parsers and Improve Error Handling ([#11148](https://github.com/n8n-io/n8n/issues/11148)) ([45274f2](https://github.com/n8n-io/n8n/commit/45274f2e7f081e194e330e1c9e6a5c26fca0b141)) + + + +# [1.64.0](https://github.com/n8n-io/n8n/compare/n8n@1.63.0...n8n@1.64.0) (2024-10-16) + + +### Bug Fixes + +* Adjust arrow button colors in dark mode ([#11248](https://github.com/n8n-io/n8n/issues/11248)) ([439132c](https://github.com/n8n-io/n8n/commit/439132c291a812d57702c94eaa12878394ac4c69)) +* **core:** Ensure error reporter does not promote `info` to `error` messages ([#11245](https://github.com/n8n-io/n8n/issues/11245)) ([a7fc7fc](https://github.com/n8n-io/n8n/commit/a7fc7fc22997acec86dc94386c95349fd018f4ae)) +* **core:** Override executions mode if `regular` during worker startup ([#11250](https://github.com/n8n-io/n8n/issues/11250)) ([c0aa28c](https://github.com/n8n-io/n8n/commit/c0aa28c6cf3f77b04e04663217c9df8e3803ed3f)) +* **core:** Wrap nodes for tool use at a suitable time ([#11238](https://github.com/n8n-io/n8n/issues/11238)) ([c2fb881](https://github.com/n8n-io/n8n/commit/c2fb881d61291209802438d95892d052f5c82d43)) +* Don't show pinned data tooltip for pinned nodes ([#11249](https://github.com/n8n-io/n8n/issues/11249)) ([c2ad156](https://github.com/n8n-io/n8n/commit/c2ad15646d326a8f71e314d54efe202a5bcdd296)) +* **editor:** Bring back the "Forgot password" link on SigninView ([#11216](https://github.com/n8n-io/n8n/issues/11216)) ([4e78c46](https://github.com/n8n-io/n8n/commit/4e78c46a7450c7fc0694369944d4fb446cef2348)) +* **editor:** Fix chat crashing when rendering output-parsed content ([#11210](https://github.com/n8n-io/n8n/issues/11210)) ([4aaebfd](https://github.com/n8n-io/n8n/commit/4aaebfd4358f590e98c453ad4e65cc2c9d0f76f8)) +* **editor:** Make submit in ChangePasswordView work again ([#11227](https://github.com/n8n-io/n8n/issues/11227)) ([4f27b39](https://github.com/n8n-io/n8n/commit/4f27b39b45b58779d363980241e6e5e11b58f5da)) +* Expressions display actual result of evaluating expression inside string ([#11257](https://github.com/n8n-io/n8n/issues/11257)) ([7f5f0a9](https://github.com/n8n-io/n8n/commit/7f5f0a9df3b3fae6e2f9787443ac1cf9415d5932)) +* **Google Ads Node:** Update to use v17 api ([#11243](https://github.com/n8n-io/n8n/issues/11243)) ([3d97f02](https://github.com/n8n-io/n8n/commit/3d97f02a8d2b6e5bc7c97c5271bed97417ecacd2)) +* **Google Calendar Node:** Fix issue with conference data types not loading ([#11185](https://github.com/n8n-io/n8n/issues/11185)) ([4012758](https://github.com/n8n-io/n8n/commit/401275884e5db0287e4eeffb3c7497dd5e024880)) +* **Google Calendar Node:** Mode to add or replace attendees in event update ([#11132](https://github.com/n8n-io/n8n/issues/11132)) ([6c6a8ef](https://github.com/n8n-io/n8n/commit/6c6a8efdea83cf7194304ce089d7b72d8f6c1a9d)) +* **HTTP Request Tool Node:** Respond with an error when receive binary response ([#11219](https://github.com/n8n-io/n8n/issues/11219)) ([0d23a7f](https://github.com/n8n-io/n8n/commit/0d23a7fb5ba41545f70c4848d30b90af91b1e7e6)) +* **MySQL Node:** Fix "Maximum call stack size exceeded" error when handling a large number of rows ([#11242](https://github.com/n8n-io/n8n/issues/11242)) ([b7ee0c4](https://github.com/n8n-io/n8n/commit/b7ee0c4087eae346bc7e5360130d6c812dbe99db)) +* **n8n Trigger Node:** Merge with Workflow Trigger node ([#11174](https://github.com/n8n-io/n8n/issues/11174)) ([6ec6b51](https://github.com/n8n-io/n8n/commit/6ec6b5197ae97eb86496effd458fcc0b9b223ef3)) +* **OpenAI Node:** Fix tool parameter parsing issue ([#11201](https://github.com/n8n-io/n8n/issues/11201)) ([5a1d81a](https://github.com/n8n-io/n8n/commit/5a1d81ad917fde5cd6a387fe2d4ec6aab6b71349)) +* **Set Node:** Fix issue with UI properties not being hidden ([#11263](https://github.com/n8n-io/n8n/issues/11263)) ([1affc27](https://github.com/n8n-io/n8n/commit/1affc27b6bf9a559061a06f92bebe8167d938665)) +* **Strava Trigger Node:** Fix issue with webhook not being deleted ([#11226](https://github.com/n8n-io/n8n/issues/11226)) ([566529c](https://github.com/n8n-io/n8n/commit/566529ca1149988a54a58b3c34bbe4d9f1add6db)) + + +### Features + +* Add tracking for node errors and update node graph ([#11060](https://github.com/n8n-io/n8n/issues/11060)) ([d3b05f1](https://github.com/n8n-io/n8n/commit/d3b05f1c54e62440666297d8e484ccd22168da48)) +* **core:** Dedupe ([#10101](https://github.com/n8n-io/n8n/issues/10101)) ([52dd2c7](https://github.com/n8n-io/n8n/commit/52dd2c76196c6895b47145c2b85a6895ce2874d4)) +* **editor:** Send workflow context to assistant store ([#11135](https://github.com/n8n-io/n8n/issues/11135)) ([fade9e4](https://github.com/n8n-io/n8n/commit/fade9e43c84a0ae1fbc80f3ee546a418970e2380)) +* **Gong Node:** New node ([#10777](https://github.com/n8n-io/n8n/issues/10777)) ([785b47f](https://github.com/n8n-io/n8n/commit/785b47feb3b83cf36aaed57123f8baca2bbab307)) + + +### Performance Improvements + +* **Google Sheets Node:** Don't load whole spreadsheet dataset to determine columns when appending data ([#11235](https://github.com/n8n-io/n8n/issues/11235)) ([26ad091](https://github.com/n8n-io/n8n/commit/26ad091f473bca4e5d3bdc257e0818be02e52db5)) + + + # [1.63.0](https://github.com/n8n-io/n8n/compare/n8n@1.62.1...n8n@1.63.0) (2024-10-09) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a67bf00ac..3f19be15e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ If you already have VS Code and Docker installed, you can click [here](https://v #### Node.js -[Node.js](https://nodejs.org/en/) version 18.10 or newer is required for development purposes. +[Node.js](https://nodejs.org/en/) version 20.15 or newer is required for development purposes. #### pnpm diff --git a/cypress/e2e/10-undo-redo.cy.ts b/cypress/e2e/10-undo-redo.cy.ts index 7e3b5ef8ad..f54c2de9fa 100644 --- a/cypress/e2e/10-undo-redo.cy.ts +++ b/cypress/e2e/10-undo-redo.cy.ts @@ -20,6 +20,7 @@ describe('Undo/Redo', () => { WorkflowPage.actions.visit(); }); + // FIXME: Canvas V2: Fix redo connections it('should undo/redo adding node in the middle', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -114,6 +115,7 @@ describe('Undo/Redo', () => { WorkflowPage.getters.nodeConnections().should('have.length', 0); }); + // FIXME: Canvas V2: Fix moving of nodes via e2e tests it('should undo/redo moving nodes', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -146,18 +148,14 @@ describe('Undo/Redo', () => { it('should undo/redo deleting a connection using context menu', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.getters.nodeConnections().realHover(); - cy.get('.connection-actions .delete') - .filter(':visible') - .should('be.visible') - .click({ force: true }); + WorkflowPage.actions.deleteNodeBetweenNodes(SCHEDULE_TRIGGER_NODE_NAME, CODE_NODE_NAME); WorkflowPage.getters.nodeConnections().should('have.length', 0); WorkflowPage.actions.hitUndo(); WorkflowPage.getters.nodeConnections().should('have.length', 1); WorkflowPage.actions.hitRedo(); WorkflowPage.getters.nodeConnections().should('have.length', 0); }); - + // FIXME: Canvas V2: Fix disconnecting by moving it('should undo/redo deleting a connection by moving it away', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -206,6 +204,7 @@ describe('Undo/Redo', () => { WorkflowPage.getters.disabledNodes().should('have.length', 2); }); + // FIXME: Canvas V2: Fix undo renaming node it('should undo/redo renaming node using keyboard shortcut', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -244,6 +243,7 @@ describe('Undo/Redo', () => { }); }); + // FIXME: Canvas V2: Figure out why moving doesn't work from e2e it('should undo/redo multiple steps', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); diff --git a/cypress/e2e/12-canvas-actions.cy.ts b/cypress/e2e/12-canvas-actions.cy.ts index 8a42521d84..e9244a1d12 100644 --- a/cypress/e2e/12-canvas-actions.cy.ts +++ b/cypress/e2e/12-canvas-actions.cy.ts @@ -16,6 +16,7 @@ describe('Canvas Actions', () => { WorkflowPage.actions.visit(); }); + // FIXME: Canvas V2: Missing execute button if no nodes it('should render canvas', () => { WorkflowPage.getters.nodeViewRoot().should('be.visible'); WorkflowPage.getters.canvasPlusButton().should('be.visible'); @@ -25,10 +26,11 @@ describe('Canvas Actions', () => { WorkflowPage.getters.executeWorkflowButton().should('be.visible'); }); + // FIXME: Canvas V2: Fix changing of connection it('should connect and disconnect a simple node', () => { WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); WorkflowPage.getters.nodeViewBackground().click(600, 200, { force: true }); - cy.get('.jtk-connector').should('have.length', 1); + WorkflowPage.getters.nodeConnections().should('have.length', 1); WorkflowPage.getters.nodeViewBackground().click(600, 400, { force: true }); WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); @@ -40,16 +42,16 @@ describe('Canvas Actions', () => { ); WorkflowPage.getters - .canvasNodeInputEndpointByName(`${EDIT_FIELDS_SET_NODE_NAME}1`) - .should('have.class', 'jtk-endpoint-connected'); + .getConnectionBetweenNodes(MANUAL_TRIGGER_NODE_DISPLAY_NAME, `${EDIT_FIELDS_SET_NODE_NAME}1`) + .should('be.visible'); - cy.get('.jtk-connector').should('have.length', 1); + WorkflowPage.getters.nodeConnections().should('have.length', 1); // Disconnect Set1 cy.drag( WorkflowPage.getters.getEndpointSelector('input', `${EDIT_FIELDS_SET_NODE_NAME}1`), [-200, 100], ); - cy.get('.jtk-connector').should('have.length', 0); + WorkflowPage.getters.nodeConnections().should('have.length', 0); }); it('should add first step', () => { @@ -74,7 +76,7 @@ describe('Canvas Actions', () => { it('should add a connected node using plus endpoint', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - cy.get('.plus-endpoint').should('be.visible').click(); + WorkflowPage.getters.canvasNodePlusEndpointByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.getters.nodeCreatorSearchBar().should('be.visible'); WorkflowPage.getters.nodeCreatorSearchBar().type(CODE_NODE_NAME); WorkflowPage.getters.nodeCreatorSearchBar().type('{enter}'); @@ -85,7 +87,7 @@ describe('Canvas Actions', () => { it('should add a connected node dragging from node creator', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - cy.get('.plus-endpoint').should('be.visible').click(); + WorkflowPage.getters.canvasNodePlusEndpointByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.getters.nodeCreatorSearchBar().should('be.visible'); WorkflowPage.getters.nodeCreatorSearchBar().type(CODE_NODE_NAME); cy.drag(WorkflowPage.getters.nodeCreatorNodeItems().first(), [100, 100], { @@ -99,7 +101,7 @@ describe('Canvas Actions', () => { it('should open a category when trying to drag and drop it on the canvas', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - cy.get('.plus-endpoint').should('be.visible').click(); + WorkflowPage.getters.canvasNodePlusEndpointByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.getters.nodeCreatorSearchBar().should('be.visible'); WorkflowPage.getters.nodeCreatorSearchBar().type(CODE_NODE_NAME); cy.drag(WorkflowPage.getters.nodeCreatorActionItems().first(), [100, 100], { @@ -114,7 +116,7 @@ describe('Canvas Actions', () => { it('should add disconnected node if nothing is selected', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); // Deselect nodes - WorkflowPage.getters.nodeViewBackground().click({ force: true }); + WorkflowPage.getters.nodeView().click({ force: true }); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); WorkflowPage.getters.canvasNodes().should('have.length', 2); WorkflowPage.getters.nodeConnections().should('have.length', 0); @@ -136,10 +138,10 @@ describe('Canvas Actions', () => { WorkflowPage.getters.nodeConnections().should('have.length', 3); WorkflowPage.getters.canvasNodeByName(EDIT_FIELDS_SET_NODE_NAME).then(($editFieldsNode) => { - const editFieldsNodeLeft = parseFloat($editFieldsNode.css('left')); + const editFieldsNodeLeft = WorkflowPage.getters.getNodeLeftPosition($editFieldsNode); WorkflowPage.getters.canvasNodeByName(HTTP_REQUEST_NODE_NAME).then(($httpNode) => { - const httpNodeLeft = parseFloat($httpNode.css('left')); + const httpNodeLeft = WorkflowPage.getters.getNodeLeftPosition($httpNode); expect(httpNodeLeft).to.be.lessThan(editFieldsNodeLeft); }); }); @@ -159,10 +161,12 @@ describe('Canvas Actions', () => { WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); WorkflowPage.getters.nodeConnections().first().realHover(); - cy.get('.connection-actions .delete').first().click({ force: true }); + WorkflowPage.actions.deleteNodeBetweenNodes(MANUAL_TRIGGER_NODE_DISPLAY_NAME, CODE_NODE_NAME); + WorkflowPage.getters.nodeConnections().should('have.length', 0); }); + // FIXME: Canvas V2: Fix disconnecting of connection by dragging it it('should delete a connection by moving it away from endpoint', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); @@ -216,10 +220,10 @@ describe('Canvas Actions', () => { WorkflowPage.actions.hitSelectAll(); WorkflowPage.actions.hitCopy(); - successToast().should('contain', 'Copied!'); + successToast().should('contain', 'Copied to clipboard'); WorkflowPage.actions.copyNode(CODE_NODE_NAME); - successToast().should('contain', 'Copied!'); + successToast().should('contain', 'Copied to clipboard'); }); it('should select/deselect all nodes', () => { @@ -231,17 +235,31 @@ describe('Canvas Actions', () => { WorkflowPage.getters.selectedNodes().should('have.length', 0); }); + // FIXME: Canvas V2: Selection via arrow keys is broken it('should select nodes using arrow keys', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); cy.wait(500); cy.get('body').type('{leftArrow}'); - WorkflowPage.getters.canvasNodes().first().should('have.class', 'jtk-drag-selected'); + const selectedCanvasNodes = () => + cy.ifCanvasVersion( + () => WorkflowPage.getters.canvasNodes(), + () => WorkflowPage.getters.canvasNodes().parent(), + ); + + cy.ifCanvasVersion( + () => selectedCanvasNodes().first().should('have.class', 'jtk-drag-selected'), + () => selectedCanvasNodes().first().should('have.class', 'selected'), + ); cy.get('body').type('{rightArrow}'); - WorkflowPage.getters.canvasNodes().last().should('have.class', 'jtk-drag-selected'); + cy.ifCanvasVersion( + () => selectedCanvasNodes().last().should('have.class', 'jtk-drag-selected'), + () => selectedCanvasNodes().last().should('have.class', 'selected'), + ); }); + // FIXME: Canvas V2: Selection via shift and arrow keys is broken it('should select nodes using shift and arrow keys', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); @@ -251,6 +269,7 @@ describe('Canvas Actions', () => { WorkflowPage.getters.selectedNodes().should('have.length', 2); }); + // FIXME: Canvas V2: Fix select & deselect it('should not break lasso selection when dragging node action buttons', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters @@ -262,6 +281,7 @@ describe('Canvas Actions', () => { WorkflowPage.actions.testLassoSelection([100, 100], [200, 200]); }); + // FIXME: Canvas V2: Fix select & deselect it('should not break lasso selection with multiple clicks on node action buttons', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.actions.testLassoSelection([100, 100], [200, 200]); diff --git a/cypress/e2e/12-canvas.cy.ts b/cypress/e2e/12-canvas.cy.ts index 4c7cccaafe..ecfb325de2 100644 --- a/cypress/e2e/12-canvas.cy.ts +++ b/cypress/e2e/12-canvas.cy.ts @@ -9,6 +9,7 @@ import { } from './../constants'; import { NDV, WorkflowExecutionsTab } from '../pages'; import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; +import { isCanvasV2 } from '../utils/workflowUtils'; const WorkflowPage = new WorkflowPageClass(); const ExecutionsTab = new WorkflowExecutionsTab(); @@ -52,15 +53,15 @@ describe('Canvas Node Manipulation and Navigation', () => { cy.reload(); cy.waitForLoad(); // Make sure outputless switch was connected correctly - cy.get( - `[data-target-node="${SWITCH_NODE_NAME}1"][data-source-node="${EDIT_FIELDS_SET_NODE_NAME}3"]`, - ).should('be.visible'); + WorkflowPage.getters + .getConnectionBetweenNodes(`${EDIT_FIELDS_SET_NODE_NAME}3`, `${SWITCH_NODE_NAME}1`) + .should('exist'); // Make sure all connections are there after reload for (let i = 0; i < desiredOutputs; i++) { const setName = `${EDIT_FIELDS_SET_NODE_NAME}${i > 0 ? i : ''}`; WorkflowPage.getters - .canvasNodeInputEndpointByName(setName) - .should('have.class', 'jtk-endpoint-connected'); + .getConnectionBetweenNodes(`${SWITCH_NODE_NAME}`, setName) + .should('exist'); } }); @@ -69,9 +70,7 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); for (let i = 0; i < 2; i++) { WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, true); - WorkflowPage.getters - .nodeViewBackground() - .click((i + 1) * 200, (i + 1) * 200, { force: true }); + WorkflowPage.getters.nodeView().click((i + 1) * 200, (i + 1) * 200, { force: true }); } WorkflowPage.actions.zoomToFit(); @@ -84,8 +83,6 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.getters.getEndpointSelector('input', `${EDIT_FIELDS_SET_NODE_NAME}1`), ); - cy.get('.rect-input-endpoint.jtk-endpoint-connected').should('have.length', 2); - // Connect Set1 and Set2 to merge cy.draganddrop( WorkflowPage.getters.getEndpointSelector('plus', EDIT_FIELDS_SET_NODE_NAME), @@ -95,20 +92,36 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.getters.getEndpointSelector('plus', `${EDIT_FIELDS_SET_NODE_NAME}1`), WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 1), ); - - cy.get('.rect-input-endpoint.jtk-endpoint-connected').should('have.length', 4); + const checkConnections = () => { + WorkflowPage.getters + .getConnectionBetweenNodes( + MANUAL_TRIGGER_NODE_DISPLAY_NAME, + `${EDIT_FIELDS_SET_NODE_NAME}1`, + ) + .should('exist'); + WorkflowPage.getters + .getConnectionBetweenNodes(EDIT_FIELDS_SET_NODE_NAME, MERGE_NODE_NAME) + .should('exist'); + WorkflowPage.getters + .getConnectionBetweenNodes(`${EDIT_FIELDS_SET_NODE_NAME}1`, MERGE_NODE_NAME) + .should('exist'); + }; + checkConnections(); // Make sure all connections are there after save & reload WorkflowPage.actions.saveWorkflowOnButtonClick(); cy.reload(); cy.waitForLoad(); - - cy.get('.rect-input-endpoint.jtk-endpoint-connected').should('have.length', 4); + checkConnections(); + // cy.get('.rect-input-endpoint.jtk-endpoint-connected').should('have.length', 4); WorkflowPage.actions.executeWorkflow(); WorkflowPage.getters.stopExecutionButton().should('not.exist'); // If the merged set nodes are connected and executed correctly, there should be 2 items in the output of merge node - cy.get('[data-label="2 items"]').should('be.visible'); + cy.ifCanvasVersion( + () => cy.get('[data-label="2 items"]').should('be.visible'), + () => cy.getByTestId('canvas-node-output-handle').contains('2 items').should('be.visible'), + ); }); it('should add nodes and check execution success', () => { @@ -120,16 +133,42 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.zoomToFit(); WorkflowPage.actions.executeWorkflow(); - cy.get('.jtk-connector.success').should('have.length', 3); - cy.get('.data-count').should('have.length', 4); - cy.get('.plus-draggable-endpoint').should('have.class', 'ep-success'); + cy.ifCanvasVersion( + () => cy.get('.jtk-connector.success').should('have.length', 3), + () => cy.get('[data-edge-status=success]').should('have.length', 3), + ); + cy.ifCanvasVersion( + () => cy.get('.data-count').should('have.length', 4), + () => cy.getByTestId('canvas-node-status-success').should('have.length', 4), + ); + + cy.ifCanvasVersion( + () => cy.get('.plus-draggable-endpoint').should('have.class', 'ep-success'), + () => cy.getByTestId('canvas-handle-plus').should('have.attr', 'data-plus-type', 'success'), + ); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); WorkflowPage.actions.zoomToFit(); - cy.get('.plus-draggable-endpoint').filter(':visible').should('not.have.class', 'ep-success'); - cy.get('.jtk-connector.success').should('have.length', 3); - cy.get('.jtk-connector').should('have.length', 4); + cy.ifCanvasVersion( + () => + cy + .get('.plus-draggable-endpoint') + .filter(':visible') + .should('not.have.class', 'ep-success'), + () => + cy.getByTestId('canvas-handle-plus').should('not.have.attr', 'data-plus-type', 'success'), + ); + + cy.ifCanvasVersion( + () => cy.get('.jtk-connector.success').should('have.length', 3), + // The new version of the canvas correctly shows executed data being passed to the input of the next node + () => cy.get('[data-edge-status=success]').should('have.length', 4), + ); + cy.ifCanvasVersion( + () => cy.get('.data-count').should('have.length', 4), + () => cy.getByTestId('canvas-node-status-success').should('have.length', 4), + ); }); it('should delete node using context menu', () => { @@ -194,19 +233,29 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.getters.canvasNodes().should('have.length', 0); }); + // FIXME: Canvas V2: Figure out how to test moving of the node it('should move node', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); + WorkflowPage.actions.zoomToFit(); WorkflowPage.getters .canvasNodes() .last() .then(($node) => { const { left, top } = $node.position(); - cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { - clickToFinish: true, - }); + + if (isCanvasV2()) { + cy.drag('.vue-flow__node', [300, 300], { + realMouse: true, + }); + } else { + cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { + clickToFinish: true, + }); + } + WorkflowPage.getters .canvasNodes() .last() @@ -218,91 +267,80 @@ describe('Canvas Node Manipulation and Navigation', () => { }); }); - it('should zoom in', () => { - WorkflowPage.getters.zoomInButton().should('be.visible').click(); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${ZOOM_IN_X1_FACTOR}, 0, 0, ${ZOOM_IN_X1_FACTOR}, 0, 0)`, + describe('Canvas Zoom Functionality', () => { + const getContainer = () => + cy.ifCanvasVersion( + () => WorkflowPage.getters.nodeView(), + () => WorkflowPage.getters.canvasViewport(), ); - WorkflowPage.getters.zoomInButton().click(); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${ZOOM_IN_X2_FACTOR}, 0, 0, ${ZOOM_IN_X2_FACTOR}, 0, 0)`, - ); - }); + const checkZoomLevel = (expectedFactor: number) => { + return getContainer().should(($nodeView) => { + const newTransform = $nodeView.css('transform'); + const newScale = parseFloat(newTransform.split(',')[0].slice(7)); - it('should zoom out', () => { - WorkflowPage.getters.zoomOutButton().should('be.visible').click(); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${ZOOM_OUT_X1_FACTOR}, 0, 0, ${ZOOM_OUT_X1_FACTOR}, 0, 0)`, - ); - WorkflowPage.getters.zoomOutButton().click(); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${ZOOM_OUT_X2_FACTOR}, 0, 0, ${ZOOM_OUT_X2_FACTOR}, 0, 0)`, - ); - }); + expect(newScale).to.be.closeTo(expectedFactor, 0.2); + }); + }; - it('should zoom using scroll or pinch gesture', () => { - WorkflowPage.actions.pinchToZoom(1, 'zoomIn'); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${PINCH_ZOOM_IN_FACTOR}, 0, 0, ${PINCH_ZOOM_IN_FACTOR}, 0, 0)`, + const zoomAndCheck = (action: 'zoomIn' | 'zoomOut', expectedFactor: number) => { + WorkflowPage.getters[`${action}Button`]().click(); + checkZoomLevel(expectedFactor); + }; + + it('should zoom in', () => { + WorkflowPage.getters.zoomInButton().should('be.visible'); + getContainer().then(($nodeView) => { + const initialTransform = $nodeView.css('transform'); + const initialScale = + initialTransform === 'none' ? 1 : parseFloat(initialTransform.split(',')[0].slice(7)); + + zoomAndCheck('zoomIn', initialScale * ZOOM_IN_X1_FACTOR); + zoomAndCheck('zoomIn', initialScale * ZOOM_IN_X2_FACTOR); + }); + }); + + it('should zoom out', () => { + zoomAndCheck('zoomOut', ZOOM_OUT_X1_FACTOR); + zoomAndCheck('zoomOut', ZOOM_OUT_X2_FACTOR); + }); + + it('should zoom using scroll or pinch gesture', () => { + WorkflowPage.actions.pinchToZoom(1, 'zoomIn'); + + // V2 Canvas is using the same zoom factor for both pinch and scroll + cy.ifCanvasVersion( + () => checkZoomLevel(PINCH_ZOOM_IN_FACTOR), + () => checkZoomLevel(ZOOM_IN_X1_FACTOR), ); - WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); - // Zoom in 1x + Zoom out 1x should reset to default (=1) - WorkflowPage.getters.nodeView().should('have.css', 'transform', 'matrix(1, 0, 0, 1, 0, 0)'); + WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); + checkZoomLevel(1); // Zoom in 1x + Zoom out 1x should reset to default (=1) - WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${PINCH_ZOOM_OUT_FACTOR}, 0, 0, ${PINCH_ZOOM_OUT_FACTOR}, 0, 0)`, + WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); + + cy.ifCanvasVersion( + () => checkZoomLevel(PINCH_ZOOM_OUT_FACTOR), + () => checkZoomLevel(ZOOM_OUT_X1_FACTOR), ); - }); + }); - it('should reset zoom', () => { - // Reset zoom should not appear until zoom level changed - WorkflowPage.getters.resetZoomButton().should('not.exist'); - WorkflowPage.getters.zoomInButton().click(); - WorkflowPage.getters.resetZoomButton().should('be.visible').click(); - WorkflowPage.getters - .nodeView() - .should( - 'have.css', - 'transform', - `matrix(${DEFAULT_ZOOM_FACTOR}, 0, 0, ${DEFAULT_ZOOM_FACTOR}, 0, 0)`, - ); - }); + it('should reset zoom', () => { + WorkflowPage.getters.resetZoomButton().should('not.exist'); + WorkflowPage.getters.zoomInButton().click(); + WorkflowPage.getters.resetZoomButton().should('be.visible').click(); + checkZoomLevel(DEFAULT_ZOOM_FACTOR); + }); - it('should zoom to fit', () => { - WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - // At this point last added node should be off-screen - WorkflowPage.getters.canvasNodes().last().should('not.be.visible'); - WorkflowPage.getters.zoomToFitButton().click(); - WorkflowPage.getters.canvasNodes().last().should('be.visible'); + it('should zoom to fit', () => { + WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); + WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); + WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); + WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); + // At this point last added node should be off-screen + WorkflowPage.getters.canvasNodes().last().should('not.be.visible'); + WorkflowPage.getters.zoomToFitButton().click(); + WorkflowPage.getters.canvasNodes().last().should('be.visible'); + }); }); it('should disable node (context menu or shortcut)', () => { @@ -426,9 +464,9 @@ describe('Canvas Node Manipulation and Navigation', () => { cy.reload(); cy.waitForLoad(); WorkflowPage.getters.canvasNodes().should('have.length', 2); - cy.get('.rect-input-endpoint.jtk-endpoint-connected').should('have.length', 1); + WorkflowPage.getters.nodeConnections().should('have.length', 1); }); - + // FIXME: Canvas V2: Credentials should show issue on the first open it('should remove unknown credentials on pasting workflow', () => { cy.fixture('workflow-with-unknown-credentials.json').then((data) => { cy.get('body').paste(JSON.stringify(data)); @@ -441,6 +479,7 @@ describe('Canvas Node Manipulation and Navigation', () => { }); }); + // FIXME: Canvas V2: Unknown nodes should still render connection endpoints it('should render connections correctly if unkown nodes are present', () => { const unknownNodeName = 'Unknown node'; cy.createFixtureWorkflow('workflow-with-unknown-nodes.json', 'Unknown nodes'); diff --git a/cypress/e2e/13-pinning.cy.ts b/cypress/e2e/13-pinning.cy.ts index 4558c44bca..4f48fa4529 100644 --- a/cypress/e2e/13-pinning.cy.ts +++ b/cypress/e2e/13-pinning.cy.ts @@ -1,3 +1,6 @@ +import { nanoid } from 'nanoid'; + +import { simpleWebhookCall, waitForWebhook } from './16-webhook-node.cy'; import { HTTP_REQUEST_NODE_NAME, MANUAL_TRIGGER_NODE_NAME, @@ -7,6 +10,7 @@ import { } from '../constants'; import { WorkflowPage, NDV } from '../pages'; import { errorToast } from '../pages/notifications'; +import { getVisiblePopper } from '../utils'; const workflowPage = new WorkflowPage(); const ndv = new NDV(); @@ -212,6 +216,42 @@ describe('Data pinning', () => { }, ); }); + + it('should show pinned data tooltip', () => { + const { callEndpoint } = simpleWebhookCall({ + method: 'GET', + webhookPath: nanoid(), + executeNow: false, + }); + + ndv.actions.close(); + workflowPage.actions.executeWorkflow(); + cy.wait(waitForWebhook); + + // hide other visible popper on workflow execute button + workflowPage.getters.canvasNodes().eq(0).click(); + + callEndpoint((response) => { + expect(response.status).to.eq(200); + getVisiblePopper().should('have.length', 1); + getVisiblePopper() + .eq(0) + .should( + 'have.text', + 'You can pin this output instead of waiting for a test event. Open node to do so.', + ); + }); + }); + + it('should not show pinned data tooltip', () => { + cy.createFixtureWorkflow('Pinned_webhook_node.json', 'Test'); + workflowPage.actions.executeWorkflow(); + + // hide other visible popper on workflow execute button + workflowPage.getters.canvasNodes().eq(0).click(); + + getVisiblePopper().should('have.length', 0); + }); }); function setExpressionOnStringValueInSet(expression: string) { diff --git a/cypress/e2e/16-form-trigger-node.cy.ts b/cypress/e2e/16-form-trigger-node.cy.ts index 0162479f7c..60fbd7c419 100644 --- a/cypress/e2e/16-form-trigger-node.cy.ts +++ b/cypress/e2e/16-form-trigger-node.cy.ts @@ -16,12 +16,14 @@ describe('n8n Form Trigger', () => { ndv.getters.parameterInput('formDescription').type('Test Form Description'); ndv.getters.parameterInput('fieldLabel').type('Test Field 1'); ndv.getters.backToCanvas().click(); - workflowPage.getters.nodeIssuesByName('n8n Form Trigger').should('not.exist'); + workflowPage.getters.nodeIssuesByName('On form submission').should('not.exist'); }); it('should fill up form fields', () => { - workflowPage.actions.addInitialNodeToCanvas('n8n Form Trigger'); - workflowPage.getters.canvasNodes().first().dblclick(); + workflowPage.actions.addInitialNodeToCanvas('n8n Form Trigger', { + isTrigger: true, + action: 'On new n8n Form event', + }); ndv.getters.parameterInput('formTitle').type('Test Form'); ndv.getters.parameterInput('formDescription').type('Test Form Description'); //fill up first field of type number @@ -96,6 +98,6 @@ describe('n8n Form Trigger', () => { .type('Your test form was successfully submitted'); ndv.getters.backToCanvas().click(); - workflowPage.getters.nodeIssuesByName('n8n Form Trigger').should('not.exist'); + workflowPage.getters.nodeIssuesByName('On form submission').should('not.exist'); }); }); diff --git a/cypress/e2e/16-webhook-node.cy.ts b/cypress/e2e/16-webhook-node.cy.ts index 9346004388..3d6c1049a2 100644 --- a/cypress/e2e/16-webhook-node.cy.ts +++ b/cypress/e2e/16-webhook-node.cy.ts @@ -9,7 +9,7 @@ const workflowPage = new WorkflowPage(); const ndv = new NDV(); const credentialsModal = new CredentialsModal(); -const waitForWebhook = 500; +export const waitForWebhook = 500; interface SimpleWebhookCallOptions { method: string; @@ -21,7 +21,7 @@ interface SimpleWebhookCallOptions { authentication?: string; } -const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { +export const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { const { authentication, method, @@ -65,15 +65,23 @@ const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { getVisibleSelect().find('.option-headline').contains(responseData).click(); } + const callEndpoint = (cb: (response: Cypress.Response) => void) => { + cy.request(method, `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then(cb); + }; + if (executeNow) { ndv.actions.execute(); cy.wait(waitForWebhook); - cy.request(method, `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { + callEndpoint((response) => { expect(response.status).to.eq(200); ndv.getters.outputPanel().contains('headers'); }); } + + return { + callEndpoint, + }; }; describe('Webhook Trigger node', () => { diff --git a/cypress/e2e/17-sharing.cy.ts b/cypress/e2e/17-sharing.cy.ts index e2af15f101..51b4a674d3 100644 --- a/cypress/e2e/17-sharing.cy.ts +++ b/cypress/e2e/17-sharing.cy.ts @@ -226,6 +226,7 @@ describe('Sharing', { disableAutoLogin: true }, () => { .should('have.length', 1) .click(); credentialsModal.getters.saveButton().click(); + credentialsModal.getters.saveButton().should('have.text', 'Saved'); credentialsModal.actions.close(); projects.getProjectTabWorkflows().click(); @@ -252,12 +253,13 @@ describe('Sharing', { disableAutoLogin: true }, () => { credentialsModal.getters.usersSelect().click(); getVisibleSelect().find('li').should('have.length', 4).first().click(); credentialsModal.getters.saveButton().click(); + credentialsModal.getters.saveButton().should('have.text', 'Saved'); credentialsModal.actions.close(); credentialsPage.getters .credentialCards() .should('have.length', 2) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('have.length', 1); }); }); diff --git a/cypress/e2e/19-execution.cy.ts b/cypress/e2e/19-execution.cy.ts index 81e11b1b63..5be2399253 100644 --- a/cypress/e2e/19-execution.cy.ts +++ b/cypress/e2e/19-execution.cy.ts @@ -1,6 +1,7 @@ import { SCHEDULE_TRIGGER_NODE_NAME, EDIT_FIELDS_SET_NODE_NAME } from '../constants'; import { NDV, WorkflowExecutionsTab, WorkflowPage as WorkflowPageClass } from '../pages'; import { clearNotifications, errorToast, successToast } from '../pages/notifications'; +import { isCanvasV2 } from '../utils/workflowUtils'; const workflowPage = new WorkflowPageClass(); const executionsTab = new WorkflowExecutionsTab(); @@ -117,15 +118,22 @@ describe('Execution', () => { .canvasNodeByName('Manual') .within(() => cy.get('.fa-check')) .should('exist'); - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-sync-alt').should('not.visible')); + + if (isCanvasV2()) { + workflowPage.getters + .canvasNodeByName('Wait') + .within(() => cy.get('.fa-sync-alt').should('not.exist')); + } else { + workflowPage.getters + .canvasNodeByName('Wait') + .within(() => cy.get('.fa-sync-alt').should('not.be.visible')); + } + workflowPage.getters .canvasNodeByName('Set') .within(() => cy.get('.fa-check').should('not.exist')); successToast().should('be.visible'); - clearNotifications(); // Clear execution data workflowPage.getters.clearExecutionDataButton().should('be.visible'); @@ -206,6 +214,7 @@ describe('Execution', () => { workflowPage.getters.clearExecutionDataButton().should('not.exist'); }); + // FIXME: Canvas V2: Webhook should show waiting state but it doesn't it('should test webhook workflow stop', () => { cy.createFixtureWorkflow('Webhook_wait_set.json'); @@ -267,9 +276,17 @@ describe('Execution', () => { .canvasNodeByName('Webhook') .within(() => cy.get('.fa-check')) .should('exist'); - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-sync-alt').should('not.visible')); + + if (isCanvasV2()) { + workflowPage.getters + .canvasNodeByName('Wait') + .within(() => cy.get('.fa-sync-alt').should('not.exist')); + } else { + workflowPage.getters + .canvasNodeByName('Wait') + .within(() => cy.get('.fa-sync-alt').should('not.be.visible')); + } + workflowPage.getters .canvasNodeByName('Set') .within(() => cy.get('.fa-check').should('not.exist')); @@ -295,6 +312,7 @@ describe('Execution', () => { }); }); + // FIXME: Canvas V2: Missing pinned states for `edge-label-wrapper` describe('connections should be colored differently for pinned data', () => { beforeEach(() => { cy.createFixtureWorkflow('Schedule_pinned.json'); diff --git a/cypress/e2e/2-credentials.cy.ts b/cypress/e2e/2-credentials.cy.ts index dbc613bd64..8ce3bc4080 100644 --- a/cypress/e2e/2-credentials.cy.ts +++ b/cypress/e2e/2-credentials.cy.ts @@ -15,7 +15,7 @@ import { TRELLO_NODE_NAME, } from '../constants'; import { CredentialsModal, CredentialsPage, NDV, WorkflowPage } from '../pages'; -import { successToast } from '../pages/notifications'; +import { errorToast, successToast } from '../pages/notifications'; import { getVisibleSelect } from '../utils'; const credentialsPage = new CredentialsPage(); @@ -278,4 +278,25 @@ describe('Credentials', () => { credentialsModal.getters.credentialAuthTypeRadioButtons().first().click(); nodeDetailsView.getters.copyInput().should('not.exist'); }); + + it('ADO-2583 should show notifications above credential modal overlay', () => { + // check error notifications because they are sticky + cy.intercept('POST', '/rest/credentials', { forceNetworkError: true }); + credentialsPage.getters.createCredentialButton().click(); + + credentialsModal.getters.newCredentialModal().should('be.visible'); + credentialsModal.getters.newCredentialTypeSelect().should('be.visible'); + credentialsModal.getters.newCredentialTypeOption('Notion API').click(); + + credentialsModal.getters.newCredentialTypeButton().click(); + credentialsModal.getters.connectionParameter('Internal Integration Secret').type('1234567890'); + + credentialsModal.actions.setName('My awesome Notion account'); + credentialsModal.getters.saveButton().click({ force: true }); + errorToast().should('have.length', 1); + errorToast().should('be.visible'); + + errorToast().should('have.css', 'z-index', '2100'); + cy.get('.el-overlay').should('have.css', 'z-index', '2001'); + }); }); diff --git a/cypress/e2e/2270-ADO-opening-webhook-ndv-marks-workflow-as-unsaved.cy.ts b/cypress/e2e/2270-ADO-opening-webhook-ndv-marks-workflow-as-unsaved.cy.ts new file mode 100644 index 0000000000..eede668e1e --- /dev/null +++ b/cypress/e2e/2270-ADO-opening-webhook-ndv-marks-workflow-as-unsaved.cy.ts @@ -0,0 +1,21 @@ +import { WEBHOOK_NODE_NAME } from '../constants'; +import { NDV, WorkflowPage } from '../pages'; + +const workflowPage = new WorkflowPage(); +const ndv = new NDV(); + +describe('ADO-2270 Save button resets on webhook node open', () => { + it('should not reset the save button if webhook node is opened and closed', () => { + workflowPage.actions.visit(); + workflowPage.actions.addInitialNodeToCanvas(WEBHOOK_NODE_NAME); + workflowPage.getters.saveButton().click(); + workflowPage.actions.openNode(WEBHOOK_NODE_NAME); + + ndv.actions.close(); + + cy.ifCanvasVersion( + () => cy.getByTestId('workflow-save-button').should('not.contain', 'Saved'), + () => cy.getByTestId('workflow-save-button').should('contain', 'Saved'), + ); + }); +}); diff --git a/cypress/e2e/2372-ado-prevent-clipping-params.cy.ts b/cypress/e2e/2372-ado-prevent-clipping-params.cy.ts new file mode 100644 index 0000000000..260c6e48c9 --- /dev/null +++ b/cypress/e2e/2372-ado-prevent-clipping-params.cy.ts @@ -0,0 +1,86 @@ +import { NDV, WorkflowPage } from '../pages'; + +const workflowPage = new WorkflowPage(); +const ndv = new NDV(); + +describe('ADO-2362 ADO-2350 NDV Prevent clipping long parameters and scrolling to expression', () => { + it('should show last parameters and open at scroll top of parameters', () => { + workflowPage.actions.visit(); + cy.createFixtureWorkflow('Test-workflow-with-long-parameters.json'); + workflowPage.actions.openNode('Schedule Trigger'); + + ndv.getters.inlineExpressionEditorInput().should('be.visible'); + + ndv.actions.close(); + + workflowPage.actions.openNode('Edit Fields1'); + + // first parameter should be visible + ndv.getters.inputLabel().eq(0).should('include.text', 'Mode'); + ndv.getters.inputLabel().eq(0).should('be.visible'); + + ndv.getters.inlineExpressionEditorInput().should('have.length', 2); + + // last parameter in view should be visible + ndv.getters.inlineExpressionEditorInput().eq(0).should('have.text', 'should be visible!'); + ndv.getters.inlineExpressionEditorInput().eq(0).should('be.visible'); + + // next parameter in view should not be visible + ndv.getters.inlineExpressionEditorInput().eq(1).should('have.text', 'not visible'); + ndv.getters.inlineExpressionEditorInput().eq(1).should('not.be.visible'); + + ndv.actions.close(); + workflowPage.actions.openNode('Schedule Trigger'); + + // first parameter (notice) should be visible + ndv.getters.nthParam(0).should('include.text', 'This workflow will run on the schedule '); + ndv.getters.inputLabel().eq(0).should('be.visible'); + + ndv.getters.inlineExpressionEditorInput().should('have.length', 2); + + // last parameter in view should be visible + ndv.getters.inlineExpressionEditorInput().eq(0).should('have.text', 'should be visible'); + ndv.getters.inlineExpressionEditorInput().eq(0).should('be.visible'); + + // next parameter in view should not be visible + ndv.getters.inlineExpressionEditorInput().eq(1).should('have.text', 'not visible'); + ndv.getters.inlineExpressionEditorInput().eq(1).should('not.be.visible'); + + ndv.actions.close(); + workflowPage.actions.openNode('Slack'); + + // first field (credentials) should be visible + ndv.getters.nodeCredentialsLabel().should('be.visible'); + + // last parameter in view should be visible + ndv.getters.inlineExpressionEditorInput().eq(0).should('have.text', 'should be visible'); + ndv.getters.inlineExpressionEditorInput().eq(0).should('be.visible'); + + // next parameter in view should not be visible + ndv.getters.inlineExpressionEditorInput().eq(1).should('have.text', 'not visible'); + ndv.getters.inlineExpressionEditorInput().eq(1).should('not.be.visible'); + }); + + it('NODE-1272 ensure expressions scrolled to top, not middle', () => { + workflowPage.actions.visit(); + cy.createFixtureWorkflow('Test-workflow-with-long-parameters.json'); + workflowPage.actions.openNode('With long expression'); + + ndv.getters.inlineExpressionEditorInput().eq(0).should('be.visible'); + // should be scrolled at top + ndv.getters + .inlineExpressionEditorInput() + .eq(0) + .find('.cm-line') + .eq(0) + .should('have.text', '1 visible!'); + ndv.getters.inlineExpressionEditorInput().eq(0).find('.cm-line').eq(0).should('be.visible'); + ndv.getters + .inlineExpressionEditorInput() + .eq(0) + .find('.cm-line') + .eq(6) + .should('have.text', '7 not visible!'); + ndv.getters.inlineExpressionEditorInput().eq(0).find('.cm-line').eq(6).should('not.be.visible'); + }); +}); diff --git a/cypress/e2e/28-debug.cy.ts b/cypress/e2e/28-debug.cy.ts index bc1f03c162..b5159951a7 100644 --- a/cypress/e2e/28-debug.cy.ts +++ b/cypress/e2e/28-debug.cy.ts @@ -117,7 +117,8 @@ describe('Debug', () => { workflowPage.getters.canvasNodes().last().find('.node-info-icon').should('be.empty'); workflowPage.getters.canvasNodes().first().dblclick(); - ndv.getters.pinDataButton().click(); + ndv.actions.unPinData(); + ndv.actions.close(); workflowPage.actions.saveWorkflowUsingKeyboardShortcut(); diff --git a/cypress/e2e/39-projects.cy.ts b/cypress/e2e/39-projects.cy.ts index 4e3bb583df..138f67838a 100644 --- a/cypress/e2e/39-projects.cy.ts +++ b/cypress/e2e/39-projects.cy.ts @@ -15,6 +15,7 @@ import { NDV, MainSidebar, } from '../pages'; +import { clearNotifications } from '../pages/notifications'; import { getVisibleDropdown, getVisibleModalOverlay, getVisibleSelect } from '../utils'; const workflowsPage = new WorkflowsPage(); @@ -448,38 +449,48 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowsPage.getters.workflowCards().should('not.have.length'); workflowsPage.getters.newWorkflowButtonCard().click(); projects.createWorkflow('Test_workflow_1.json', 'Workflow in Home project'); + clearNotifications(); projects.getHomeButton().click(); projects.getProjectTabCredentials().should('be.visible').click(); credentialsPage.getters.emptyListCreateCredentialButton().click(); projects.createCredential('Credential in Home project'); + clearNotifications(); + // Create a project and add a credential and a workflow to it projects.createProject('Project 1'); + clearNotifications(); projects.getProjectTabCredentials().click(); credentialsPage.getters.emptyListCreateCredentialButton().click(); projects.createCredential('Credential in Project 1'); + clearNotifications(); projects.getProjectTabWorkflows().click(); workflowsPage.getters.newWorkflowButtonCard().click(); projects.createWorkflow('Test_workflow_1.json', 'Workflow in Project 1'); + clearNotifications(); + // Create another project and add a credential and a workflow to it projects.createProject('Project 2'); + clearNotifications(); projects.getProjectTabCredentials().click(); credentialsPage.getters.emptyListCreateCredentialButton().click(); projects.createCredential('Credential in Project 2'); + clearNotifications(); projects.getProjectTabWorkflows().click(); workflowsPage.getters.newWorkflowButtonCard().click(); projects.createWorkflow('Test_workflow_1.json', 'Workflow in Project 2'); + clearNotifications(); - // Move the workflow owned by me from Home to Project 1 + // Move the workflow Personal from Home to Project 1 projects.getHomeButton().click(); workflowsPage.getters .workflowCards() .should('have.length', 3) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('exist'); workflowsPage.getters.workflowCardActions('Workflow in Home project').click(); workflowsPage.getters.workflowMoveButton().click(); @@ -496,11 +507,12 @@ describe('Projects', { disableAutoLogin: true }, () => { .filter(':contains("Project 1")') .click(); projects.getResourceMoveModal().find('button:contains("Move workflow")').click(); + clearNotifications(); workflowsPage.getters .workflowCards() .should('have.length', 3) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('not.exist'); // Move the workflow from Project 1 to Project 2 @@ -527,6 +539,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowsPage.getters.workflowCards().should('have.length', 2); workflowsPage.getters.workflowCardActions('Workflow in Home project').click(); workflowsPage.getters.workflowMoveButton().click(); + clearNotifications(); projects .getResourceMoveModal() @@ -566,10 +579,11 @@ describe('Projects', { disableAutoLogin: true }, () => { .click(); projects.getResourceMoveModal().find('button:contains("Move workflow")').click(); + clearNotifications(); workflowsPage.getters .workflowCards() .should('have.length', 3) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('have.length', 1); // Move the credential from Project 1 to Project 2 @@ -591,7 +605,7 @@ describe('Projects', { disableAutoLogin: true }, () => { .filter(':contains("Project 2")') .click(); projects.getResourceMoveModal().find('button:contains("Move credential")').click(); - + clearNotifications(); credentialsPage.getters.credentialCards().should('not.have.length'); // Move the credential from Project 2 to admin user @@ -637,10 +651,12 @@ describe('Projects', { disableAutoLogin: true }, () => { .click(); projects.getResourceMoveModal().find('button:contains("Move credential")').click(); + clearNotifications(); + credentialsPage.getters .credentialCards() .should('have.length', 3) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('have.length', 2); // Move the credential from admin user back to its original project (Project 1) @@ -699,7 +715,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowsPage.getters .workflowCards() .should('have.length', 1) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('exist'); workflowsPage.getters.workflowCardActions('My workflow').click(); workflowsPage.getters.workflowMoveButton().click(); @@ -720,7 +736,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowsPage.getters .workflowCards() .should('have.length', 1) - .filter(':contains("Owned by me")') + .filter(':contains("Personal")') .should('not.exist'); //Log out with instance owner and log in with the member user diff --git a/cypress/e2e/40-manual-partial-execution.cy.ts b/cypress/e2e/40-manual-partial-execution.cy.ts index 5fe31b56ad..2eb129475f 100644 --- a/cypress/e2e/40-manual-partial-execution.cy.ts +++ b/cypress/e2e/40-manual-partial-execution.cy.ts @@ -23,6 +23,7 @@ describe('Manual partial execution', () => { canvas.actions.openNode('Webhook1'); ndv.getters.nodeRunSuccessIndicator().should('exist'); + ndv.getters.nodeRunTooltipIndicator().should('exist'); ndv.getters.outputRunSelector().should('not.exist'); // single run }); }); diff --git a/cypress/e2e/5-ndv.cy.ts b/cypress/e2e/5-ndv.cy.ts index a591d62895..f2ccccb6ab 100644 --- a/cypress/e2e/5-ndv.cy.ts +++ b/cypress/e2e/5-ndv.cy.ts @@ -133,9 +133,10 @@ describe('NDV', () => { "An expression here won't work because it uses .item and n8n can't figure out the matching item.", ); ndv.getters.nodeRunErrorIndicator().should('be.visible'); + ndv.getters.nodeRunTooltipIndicator().should('be.visible'); // The error details should be hidden behind a tooltip - ndv.getters.nodeRunErrorIndicator().should('not.contain', 'Start Time'); - ndv.getters.nodeRunErrorIndicator().should('not.contain', 'Execution Time'); + ndv.getters.nodeRunTooltipIndicator().should('not.contain', 'Start Time'); + ndv.getters.nodeRunTooltipIndicator().should('not.contain', 'Execution Time'); }); it('should save workflow using keyboard shortcut from NDV', () => { @@ -617,8 +618,10 @@ describe('NDV', () => { // Should not show run info before execution ndv.getters.nodeRunSuccessIndicator().should('not.exist'); ndv.getters.nodeRunErrorIndicator().should('not.exist'); + ndv.getters.nodeRunTooltipIndicator().should('not.exist'); ndv.getters.nodeExecuteButton().click(); ndv.getters.nodeRunSuccessIndicator().should('exist'); + ndv.getters.nodeRunTooltipIndicator().should('exist'); }); it('should properly show node execution indicator for multiple nodes', () => { @@ -630,6 +633,7 @@ describe('NDV', () => { // Manual tigger node should show success indicator workflowPage.actions.openNode('When clicking ‘Test workflow’'); ndv.getters.nodeRunSuccessIndicator().should('exist'); + ndv.getters.nodeRunTooltipIndicator().should('exist'); // Code node should show error ndv.getters.backToCanvas().click(); workflowPage.actions.openNode('Code'); diff --git a/cypress/e2e/6-code-node.cy.ts b/cypress/e2e/6-code-node.cy.ts index 5a6182c25a..5bc7d05ee2 100644 --- a/cypress/e2e/6-code-node.cy.ts +++ b/cypress/e2e/6-code-node.cy.ts @@ -162,21 +162,21 @@ return [] cy.get('#tab-code').should('have.class', 'is-active'); }); - it('should show error based on status code', () => { - const prompt = nanoid(20); - cy.get('#tab-ask-ai').click(); - ndv.actions.executePrevious(); + const handledCodes = [ + { code: 400, message: 'Code generation failed due to an unknown reason' }, + { code: 413, message: 'Your workflow data is too large for AI to process' }, + { code: 429, message: "We've hit our rate limit with our AI partner" }, + { code: 500, message: 'Code generation failed due to an unknown reason' }, + ]; - cy.getByTestId('ask-ai-prompt-input').type(prompt); + handledCodes.forEach(({ code, message }) => { + it(`should show error based on status code ${code}`, () => { + const prompt = nanoid(20); + cy.get('#tab-ask-ai').click(); + ndv.actions.executePrevious(); - const handledCodes = [ - { code: 400, message: 'Code generation failed due to an unknown reason' }, - { code: 413, message: 'Your workflow data is too large for AI to process' }, - { code: 429, message: "We've hit our rate limit with our AI partner" }, - { code: 500, message: 'Code generation failed due to an unknown reason' }, - ]; + cy.getByTestId('ask-ai-prompt-input').type(prompt); - handledCodes.forEach(({ code, message }) => { cy.intercept('POST', '/rest/ai/ask-ai', { statusCode: code, status: code, diff --git a/cypress/fixtures/Pinned_webhook_node.json b/cypress/fixtures/Pinned_webhook_node.json new file mode 100644 index 0000000000..eb98b17351 --- /dev/null +++ b/cypress/fixtures/Pinned_webhook_node.json @@ -0,0 +1,39 @@ +{ + "nodes": [ + { + "parameters": { + "path": "FwrbSiaua2Xmvn6-Z-7CQ", + "options": {} + }, + "id": "8fcc7e5f-2cef-4938-9564-eea504c20aa0", + "name": "Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 360, + 220 + ], + "webhookId": "9c778f2a-e882-46ed-a0e4-c8e2f76ccd65" + } + ], + "connections": {}, + "pinData": { + "Webhook": [ + { + "headers": { + "connection": "keep-alive", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "accept": "*/*", + "cookie": "n8n-auth=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjNiM2FhOTE5LWRhZDgtNDE5MS1hZWZiLTlhZDIwZTZkMjJjNiIsImhhc2giOiJ1ZVAxR1F3U2paIiwiaWF0IjoxNzI4OTE1NTQyLCJleHAiOjE3Mjk1MjAzNDJ9.fV02gpUnSiUoMxHwfB0npBjcjct7Mv9vGfj-jRTT3-I", + "host": "localhost:5678", + "accept-encoding": "gzip, deflate" + }, + "params": {}, + "query": {}, + "body": {}, + "webhookUrl": "http://localhost:5678/webhook-test/FwrbSiaua2Xmvn6-Z-7CQ", + "executionMode": "test" + } + ] + } +} diff --git a/cypress/fixtures/Test-workflow-with-long-parameters.json b/cypress/fixtures/Test-workflow-with-long-parameters.json new file mode 100644 index 0000000000..d4d052f6f0 --- /dev/null +++ b/cypress/fixtures/Test-workflow-with-long-parameters.json @@ -0,0 +1,150 @@ +{ + "meta": { + "instanceId": "777c68374367604fdf2a0bcfe9b1b574575ddea61aa8268e4bf034434bd7c894" + }, + "nodes": [ + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "0effebfc-fa8c-4d41-8a37-6d5695dfc9ee", + "name": "test", + "value": "test", + "type": "string" + }, + { + "id": "beb8723f-6333-4186-ab88-41d4e2338866", + "name": "test", + "value": "test", + "type": "string" + }, + { + "id": "85095836-4e94-442f-9270-e1a89008c129", + "name": "test", + "value": "test", + "type": "string" + }, + { + "id": "b6163f8a-bca6-4364-8b38-182df37c55cd", + "name": "=should be visible!", + "value": "=not visible", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "950fcdc1-9e92-410f-8377-d4240e9bf6ff", + "name": "Edit Fields1", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 680, + 460 + ] + }, + { + "parameters": { + "messageType": "block", + "blocksUi": "blocks", + "text": "=should be visible", + "otherOptions": { + "sendAsUser": "=not visible" + } + }, + "id": "dcf7410d-0f8e-4cdb-9819-ae275558bdaa", + "name": "Slack", + "type": "n8n-nodes-base.slack", + "typeVersion": 2.2, + "position": [ + 900, + 460 + ], + "webhookId": "002b502e-31e5-4fdb-ac43-a56cfde8f82a" + }, + { + "parameters": { + "rule": { + "interval": [ + {}, + { + "field": "=should be visible" + }, + { + "field": "=not visible" + } + ] + } + }, + "id": "4c948a3f-19d4-4b08-a8be-f7d2964a21f4", + "name": "Schedule Trigger", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 460, + 460 + ] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "5dcaab37-1146-49c6-97a3-3b2f73483270", + "name": "object", + "value": "=1 visible!\n2 {\n3 \"str\": \"two\",\n4 \"str_date\": \"{{ $now }}\",\n5 \"str_int\": \"1\",\n6 \"str_float\": \"1.234\",\n7 not visible!\n \"str_bool\": \"true\",\n \"str_email\": \"david@thedavid.com\",\n \"str_with_email\":\"My email is david@n8n.io\",\n \"str_json_single\":\"{'one':'two'}\",\n \"str_json_double\":\"{\\\"one\\\":\\\"two\\\"}\",\n \"bool\": true,\n \"list\": [1, 2, 3],\n \"decimal\": 1.234,\n \"timestamp1\": 1708695471,\n \"timestamp2\": 1708695471000,\n \"timestamp3\": 1708695471000000,\n \"num_one\": 1\n}", + "type": "object" + } + ] + }, + "includeOtherFields": true, + "options": {} + }, + "id": "a41dfb0d-38aa-42d2-b3e2-1854090bd319", + "name": "With long expression", + "type": "n8n-nodes-base.set", + "typeVersion": 3.3, + "position": [ + 1100, + 460 + ] + } + ], + "connections": { + "Edit Fields1": { + "main": [ + [ + { + "node": "Slack", + "type": "main", + "index": 0 + } + ] + ] + }, + "Slack": { + "main": [ + [ + { + "node": "With long expression", + "type": "main", + "index": 0 + } + ] + ] + }, + "Schedule Trigger": { + "main": [ + [ + { + "node": "Edit Fields1", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {} +} diff --git a/cypress/pages/ndv.ts b/cypress/pages/ndv.ts index cae1fb47b0..4504552e26 100644 --- a/cypress/pages/ndv.ts +++ b/cypress/pages/ndv.ts @@ -20,7 +20,8 @@ export class NDV extends BasePage { outputDataContainer: () => this.getters.outputPanel().findChildByTestId('ndv-data-container'), outputDisplayMode: () => this.getters.outputPanel().findChildByTestId('ndv-run-data-display-mode').first(), - pinDataButton: () => cy.getByTestId('ndv-pin-data'), + pinDataButton: () => this.getters.outputPanel().findChildByTestId('ndv-pin-data'), + unpinDataLink: () => this.getters.outputPanel().findChildByTestId('ndv-unpin-data'), editPinnedDataButton: () => cy.getByTestId('ndv-edit-pinned-data'), pinnedDataEditor: () => this.getters.outputPanel().find('.cm-editor .cm-scroller .cm-content'), runDataPaneHeader: () => cy.getByTestId('run-data-pane-header'), @@ -63,6 +64,7 @@ export class NDV extends BasePage { nodeRenameInput: () => cy.getByTestId('node-rename-input'), executePrevious: () => cy.getByTestId('execute-previous-node'), httpRequestNotice: () => cy.getByTestId('node-parameters-http-notice'), + nodeCredentialsLabel: () => cy.getByTestId('credentials-label'), nthParam: (n: number) => cy.getByTestId('node-parameters').find('.parameter-item').eq(n), inputRunSelector: () => this.getters.inputPanel().findChildByTestId('run-selector'), inputLinkRun: () => this.getters.inputPanel().findChildByTestId('link-run'), @@ -130,8 +132,9 @@ export class NDV extends BasePage { codeEditorFullscreenButton: () => cy.getByTestId('code-editor-fullscreen-button'), codeEditorDialog: () => cy.getByTestId('code-editor-fullscreen'), codeEditorFullscreen: () => this.getters.codeEditorDialog().find('.cm-content'), - nodeRunSuccessIndicator: () => cy.getByTestId('node-run-info-success'), - nodeRunErrorIndicator: () => cy.getByTestId('node-run-info-danger'), + nodeRunTooltipIndicator: () => cy.getByTestId('node-run-info'), + nodeRunSuccessIndicator: () => cy.getByTestId('node-run-status-success'), + nodeRunErrorIndicator: () => cy.getByTestId('node-run-status-danger'), nodeRunErrorMessage: () => cy.getByTestId('node-error-message'), nodeRunErrorDescription: () => cy.getByTestId('node-error-description'), fixedCollectionParameter: (paramName: string) => @@ -146,6 +149,9 @@ export class NDV extends BasePage { pinData: () => { this.getters.pinDataButton().click({ force: true }); }, + unPinData: () => { + this.getters.unpinDataLink().click({ force: true }); + }, editPinnedData: () => { this.getters.editPinnedDataButton().click(); }, diff --git a/cypress/pages/workflow.ts b/cypress/pages/workflow.ts index 89186ee34e..cd1e7d9462 100644 --- a/cypress/pages/workflow.ts +++ b/cypress/pages/workflow.ts @@ -2,7 +2,7 @@ import { BasePage } from './base'; import { NodeCreator } from './features/node-creator'; import { META_KEY } from '../constants'; import { getVisibleSelect } from '../utils'; -import { getUniqueWorkflowName } from '../utils/workflowUtils'; +import { getUniqueWorkflowName, isCanvasV2 } from '../utils/workflowUtils'; const nodeCreator = new NodeCreator(); export class WorkflowPage extends BasePage { @@ -27,7 +27,11 @@ export class WorkflowPage extends BasePage { nodeCreatorSearchBar: () => cy.getByTestId('node-creator-search-bar'), nodeCreatorPlusButton: () => cy.getByTestId('node-creator-plus-button'), canvasPlusButton: () => cy.getByTestId('canvas-plus-button'), - canvasNodes: () => cy.getByTestId('canvas-node'), + canvasNodes: () => + cy.ifCanvasVersion( + () => cy.getByTestId('canvas-node'), + () => cy.getByTestId('canvas-node').not('[data-node-type="n8n-nodes-internal.addNodes"]'), + ), canvasNodeByName: (nodeName: string) => this.getters.canvasNodes().filter(`:contains(${nodeName})`), nodeIssuesByName: (nodeName: string) => @@ -37,6 +41,17 @@ export class WorkflowPage extends BasePage { .should('have.length.greaterThan', 0) .findChildByTestId('node-issues'), getEndpointSelector: (type: 'input' | 'output' | 'plus', nodeName: string, index = 0) => { + if (isCanvasV2()) { + if (type === 'input') { + return `[data-test-id="canvas-node-input-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"]`; + } + if (type === 'output') { + return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"]`; + } + if (type === 'plus') { + return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"] [data-test-id="canvas-handle-plus"] .clickable`; + } + } return `[data-endpoint-name='${nodeName}'][data-endpoint-type='${type}'][data-input-index='${index}']`; }, canvasNodeInputEndpointByName: (nodeName: string, index = 0) => { @@ -46,7 +61,15 @@ export class WorkflowPage extends BasePage { return cy.get(this.getters.getEndpointSelector('output', nodeName, index)); }, canvasNodePlusEndpointByName: (nodeName: string, index = 0) => { - return cy.get(this.getters.getEndpointSelector('plus', nodeName, index)); + return cy.ifCanvasVersion( + () => cy.get(this.getters.getEndpointSelector('plus', nodeName, index)), + () => + cy + .get( + `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"] .clickable`, + ) + .eq(index), + ); }, activatorSwitch: () => cy.getByTestId('workflow-activate-switch'), workflowMenu: () => cy.getByTestId('workflow-menu'), @@ -56,13 +79,29 @@ export class WorkflowPage extends BasePage { expressionModalInput: () => cy.getByTestId('expression-modal-input').find('[role=textbox]'), expressionModalOutput: () => cy.getByTestId('expression-modal-output'), - nodeViewRoot: () => cy.getByTestId('node-view-root'), + nodeViewRoot: () => + cy.ifCanvasVersion( + () => cy.getByTestId('node-view-root'), + () => this.getters.nodeView(), + ), copyPasteInput: () => cy.getByTestId('hidden-copy-paste'), - nodeConnections: () => cy.get('.jtk-connector'), + nodeConnections: () => + cy.ifCanvasVersion( + () => cy.get('.jtk-connector'), + () => cy.getByTestId('edge-label-wrapper'), + ), zoomToFitButton: () => cy.getByTestId('zoom-to-fit'), nodeEndpoints: () => cy.get('.jtk-endpoint-connected'), - disabledNodes: () => cy.get('.node-box.disabled'), - selectedNodes: () => this.getters.canvasNodes().filter('.jtk-drag-selected'), + disabledNodes: () => + cy.ifCanvasVersion( + () => cy.get('.node-box.disabled'), + () => cy.get('[data-test-id="canvas-trigger-node"][class*="disabled"]'), + ), + selectedNodes: () => + cy.ifCanvasVersion( + () => this.getters.canvasNodes().filter('.jtk-drag-selected'), + () => this.getters.canvasNodes().parent().filter('.selected'), + ), // Workflow menu items workflowMenuItemDuplicate: () => cy.getByTestId('workflow-menu-item-duplicate'), workflowMenuItemDownload: () => cy.getByTestId('workflow-menu-item-download'), @@ -92,8 +131,21 @@ export class WorkflowPage extends BasePage { shareButton: () => cy.getByTestId('workflow-share-button'), duplicateWorkflowModal: () => cy.getByTestId('duplicate-modal'), - nodeViewBackground: () => cy.getByTestId('node-view-background'), - nodeView: () => cy.getByTestId('node-view'), + nodeViewBackground: () => + cy.ifCanvasVersion( + () => cy.getByTestId('node-view-background'), + () => cy.getByTestId('canvas'), + ), + nodeView: () => + cy.ifCanvasVersion( + () => cy.getByTestId('node-view'), + () => cy.get('[data-test-id="canvas-wrapper"]'), + ), + canvasViewport: () => + cy.ifCanvasVersion( + () => cy.getByTestId('node-view'), + () => cy.get('.vue-flow__transformationpane.vue-flow__container'), + ), inlineExpressionEditorInput: () => cy.getByTestId('inline-expression-editor-input').find('[role=textbox]'), inlineExpressionEditorOutput: () => cy.getByTestId('inline-expression-editor-output'), @@ -115,12 +167,26 @@ export class WorkflowPage extends BasePage { ndvParameters: () => cy.getByTestId('parameter-item'), nodeCredentialsLabel: () => cy.getByTestId('credentials-label'), getConnectionBetweenNodes: (sourceNodeName: string, targetNodeName: string) => - cy.get( - `.jtk-connector[data-source-node="${sourceNodeName}"][data-target-node="${targetNodeName}"]`, + cy.ifCanvasVersion( + () => + cy.get( + `.jtk-connector[data-source-node="${sourceNodeName}"][data-target-node="${targetNodeName}"]`, + ), + () => + cy.get( + `[data-test-id="edge-label-wrapper"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"]`, + ), ), getConnectionActionsBetweenNodes: (sourceNodeName: string, targetNodeName: string) => - cy.get( - `.connection-actions[data-source-node="${sourceNodeName}"][data-target-node="${targetNodeName}"]`, + cy.ifCanvasVersion( + () => + cy.get( + `.connection-actions[data-source-node="${sourceNodeName}"][data-target-node="${targetNodeName}"]`, + ), + () => + cy.get( + `[data-test-id="edge-label-wrapper"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"] [data-test-id="canvas-edge-toolbar"]`, + ), ), addStickyButton: () => cy.getByTestId('add-sticky-button'), stickies: () => cy.getByTestId('sticky'), @@ -128,6 +194,18 @@ export class WorkflowPage extends BasePage { workflowHistoryButton: () => cy.getByTestId('workflow-history-button'), colors: () => cy.getByTestId('color'), contextMenuAction: (action: string) => cy.getByTestId(`context-menu-item-${action}`), + getNodeLeftPosition: (element: JQuery) => { + if (isCanvasV2()) { + return parseFloat(element.parent().css('transform').split(',')[4]); + } + return parseFloat(element.css('left')); + }, + getNodeTopPosition: (element: JQuery) => { + if (isCanvasV2()) { + return parseFloat(element.parent().css('transform').split(',')[5]); + } + return parseFloat(element.css('top')); + }, }; actions = { @@ -332,7 +410,7 @@ export class WorkflowPage extends BasePage { pinchToZoom: (steps: number, mode: 'zoomIn' | 'zoomOut' = 'zoomIn') => { cy.window().then((win) => { // Pinch-to-zoom simulates a 'wheel' event with ctrlKey: true (same as zooming by scrolling) - this.getters.nodeViewBackground().trigger('wheel', { + this.getters.nodeView().trigger('wheel', { force: true, bubbles: true, ctrlKey: true, @@ -391,9 +469,12 @@ export class WorkflowPage extends BasePage { action?: string, ) => { this.getters.getConnectionBetweenNodes(sourceNodeName, targetNodeName).first().realHover(); - this.getters - .getConnectionActionsBetweenNodes(sourceNodeName, targetNodeName) - .find('.add') + const connectionsBetweenNodes = () => + this.getters.getConnectionActionsBetweenNodes(sourceNodeName, targetNodeName); + cy.ifCanvasVersion( + () => connectionsBetweenNodes().find('.add'), + () => connectionsBetweenNodes().get('[data-test-id="add-connection-button"]'), + ) .first() .click({ force: true }); @@ -401,9 +482,12 @@ export class WorkflowPage extends BasePage { }, deleteNodeBetweenNodes: (sourceNodeName: string, targetNodeName: string) => { this.getters.getConnectionBetweenNodes(sourceNodeName, targetNodeName).first().realHover(); - this.getters - .getConnectionActionsBetweenNodes(sourceNodeName, targetNodeName) - .find('.delete') + const connectionsBetweenNodes = () => + this.getters.getConnectionActionsBetweenNodes(sourceNodeName, targetNodeName); + cy.ifCanvasVersion( + () => connectionsBetweenNodes().find('.delete'), + () => connectionsBetweenNodes().get('[data-test-id="delete-connection-button"]'), + ) .first() .click({ force: true }); }, diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index 35f100fded..6cad68b34f 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -10,7 +10,7 @@ import { N8N_AUTH_COOKIE, } from '../constants'; import { WorkflowPage } from '../pages'; -import { getUniqueWorkflowName } from '../utils/workflowUtils'; +import { getUniqueWorkflowName, isCanvasV2 } from '../utils/workflowUtils'; Cypress.Commands.add('setAppDate', (targetDate: number | Date) => { cy.window().then((win) => { @@ -26,6 +26,10 @@ Cypress.Commands.add('getByTestId', (selector, ...args) => { return cy.get(`[data-test-id="${selector}"]`, ...args); }); +Cypress.Commands.add('ifCanvasVersion', (getterV1, getterV2) => { + return isCanvasV2() ? getterV2() : getterV1(); +}); + Cypress.Commands.add( 'createFixtureWorkflow', (fixtureKey: string, workflowName = getUniqueWorkflowName()) => { @@ -70,6 +74,10 @@ Cypress.Commands.add('signin', ({ email, password }) => { }) .then((response) => { Cypress.env('currentUserId', response.body.data.id); + + cy.window().then((win) => { + win.localStorage.setItem('NodeView.switcher.discovered', 'true'); // @TODO Remove this once the switcher is removed + }); }); }); }); diff --git a/cypress/support/e2e.ts b/cypress/support/e2e.ts index 0fe782499d..4261cb4b63 100644 --- a/cypress/support/e2e.ts +++ b/cypress/support/e2e.ts @@ -20,6 +20,11 @@ beforeEach(() => { win.localStorage.setItem('N8N_THEME', 'light'); win.localStorage.setItem('N8N_AUTOCOMPLETE_ONBOARDED', 'true'); win.localStorage.setItem('N8N_MAPPING_ONBOARDED', 'true'); + + const nodeViewVersion = Cypress.env('NODE_VIEW_VERSION'); + if (nodeViewVersion) { + win.localStorage.setItem('NodeView.version', nodeViewVersion); + } }); cy.intercept('GET', '/rest/settings', (req) => { diff --git a/cypress/support/index.ts b/cypress/support/index.ts index a5f1caf5b2..2fd1faeb22 100644 --- a/cypress/support/index.ts +++ b/cypress/support/index.ts @@ -28,6 +28,7 @@ declare global { selector: string, ...args: Array | undefined> ): Chainable>; + ifCanvasVersion(getterV1: () => T1, getterV2: () => T2): T1 | T2; findChildByTestId(childTestId: string): Chainable>; /** * Creates a workflow from the given fixture and optionally renames it. diff --git a/cypress/utils/workflowUtils.ts b/cypress/utils/workflowUtils.ts index 5001dbe1b6..0c91a097bd 100644 --- a/cypress/utils/workflowUtils.ts +++ b/cypress/utils/workflowUtils.ts @@ -3,3 +3,7 @@ import { nanoid } from 'nanoid'; export function getUniqueWorkflowName(workflowNamePrefix?: string) { return workflowNamePrefix ? `${workflowNamePrefix} ${nanoid(12)}` : nanoid(12); } + +export function isCanvasV2() { + return Cypress.env('NODE_VIEW_VERSION') === 2; +} diff --git a/docker/images/n8n-custom/Dockerfile b/docker/images/n8n-custom/Dockerfile index ba271017d1..78eedaa2c3 100644 --- a/docker/images/n8n-custom/Dockerfile +++ b/docker/images/n8n-custom/Dockerfile @@ -31,6 +31,30 @@ WORKDIR /home/node COPY --from=builder /compiled /usr/local/lib/node_modules/n8n COPY docker/images/n8n/docker-entrypoint.sh / +# Setup the Task Runner Launcher +ARG TARGETPLATFORM +ARG LAUNCHER_VERSION=0.1.1 +ENV N8N_RUNNERS_MODE=internal_launcher \ + N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher +COPY docker/images/n8n/n8n-task-runners.json /etc/n8n-task-runners.json +# First, download, verify, then extract the launcher binary +# Second, chmod with 4555 to allow the use of setuid +# Third, create a new user and group to execute the Task Runners under +RUN \ + if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \ + elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \ + mkdir /launcher-temp && \ + cd /launcher-temp && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \ + cd - && \ + rm -r /launcher-temp && \ + chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \ + addgroup -g 2000 task-runner && \ + adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner + RUN \ cd /usr/local/lib/node_modules/n8n && \ npm rebuild sqlite3 && \ diff --git a/docker/images/n8n/Dockerfile b/docker/images/n8n/Dockerfile index 2da1bc1f47..8a94d0c9ec 100644 --- a/docker/images/n8n/Dockerfile +++ b/docker/images/n8n/Dockerfile @@ -22,6 +22,30 @@ RUN set -eux; \ find /usr/local/lib/node_modules/n8n -type f -name "*.ts" -o -name "*.js.map" -o -name "*.vue" | xargs rm -f && \ rm -rf /root/.npm +# Setup the Task Runner Launcher +ARG TARGETPLATFORM +ARG LAUNCHER_VERSION=0.1.1 +ENV N8N_RUNNERS_MODE=internal_launcher \ + N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher +COPY n8n-task-runners.json /etc/n8n-task-runners.json +# First, download, verify, then extract the launcher binary +# Second, chmod with 4555 to allow the use of setuid +# Third, create a new user and group to execute the Task Runners under +RUN \ + if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \ + elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \ + mkdir /launcher-temp && \ + cd /launcher-temp && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \ + cd - && \ + rm -r /launcher-temp && \ + chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \ + addgroup -g 2000 task-runner && \ + adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner + COPY docker-entrypoint.sh / RUN \ diff --git a/docker/images/n8n/n8n-task-runners.json b/docker/images/n8n/n8n-task-runners.json new file mode 100644 index 0000000000..699794d504 --- /dev/null +++ b/docker/images/n8n/n8n-task-runners.json @@ -0,0 +1,22 @@ +{ + "task-runners": [ + { + "runner-type": "javascript", + "workdir": "/home/task-runner", + "command": "/usr/local/bin/node", + "args": ["/usr/local/lib/node_modules/n8n/node_modules/@n8n/task-runner/dist/start.js"], + "allowed-env": [ + "PATH", + "N8N_RUNNERS_GRANT_TOKEN", + "N8N_RUNNERS_N8N_URI", + "N8N_RUNNERS_MAX_PAYLOAD", + "N8N_RUNNERS_MAX_CONCURRENCY", + "NODE_FUNCTION_ALLOW_BUILTIN", + "NODE_FUNCTION_ALLOW_EXTERNAL", + "NODE_OPTIONS" + ], + "uid": 2000, + "gid": 2000 + } + ] +} diff --git a/package.json b/package.json index ee888f53dd..09c576a8c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "n8n-monorepo", - "version": "1.63.0", + "version": "1.65.0", "private": true, "engines": { "node": ">=20.15", @@ -43,6 +43,7 @@ "@biomejs/biome": "^1.9.0", "@n8n_io/eslint-config": "workspace:*", "@types/jest": "^29.5.3", + "@types/node": "*", "@types/supertest": "^6.0.2", "jest": "^29.6.2", "jest-environment-jsdom": "^29.6.2", @@ -57,8 +58,8 @@ "run-script-os": "^1.0.7", "supertest": "^7.0.0", "ts-jest": "^29.1.1", - "tsc-alias": "^1.8.7", - "tsc-watch": "^6.0.4", + "tsc-alias": "^1.8.10", + "tsc-watch": "^6.2.0", "turbo": "2.1.2", "typescript": "*", "zx": "^8.1.4" @@ -87,7 +88,8 @@ "pyodide@0.23.4": "patches/pyodide@0.23.4.patch", "@types/express-serve-static-core@4.17.43": "patches/@types__express-serve-static-core@4.17.43.patch", "@types/ws@8.5.4": "patches/@types__ws@8.5.4.patch", - "@types/uuencode@0.0.3": "patches/@types__uuencode@0.0.3.patch" + "@types/uuencode@0.0.3": "patches/@types__uuencode@0.0.3.patch", + "@langchain/core@0.3.3": "patches/@langchain__core@0.3.3.patch" } } } diff --git a/packages/@n8n/api-types/package.json b/packages/@n8n/api-types/package.json index e2614bcf68..b3d28f18cc 100644 --- a/packages/@n8n/api-types/package.json +++ b/packages/@n8n/api-types/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/api-types", - "version": "0.4.0", + "version": "0.5.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", @@ -21,6 +21,7 @@ "dist/**/*" ], "devDependencies": { + "@n8n/config": "workspace:*", "n8n-workflow": "workspace:*" }, "dependencies": { diff --git a/packages/@n8n/api-types/src/frontend-settings.ts b/packages/@n8n/api-types/src/frontend-settings.ts index 5084344aeb..6b2f3231d3 100644 --- a/packages/@n8n/api-types/src/frontend-settings.ts +++ b/packages/@n8n/api-types/src/frontend-settings.ts @@ -1,3 +1,4 @@ +import type { FrontendBetaFeatures } from '@n8n/config'; import type { ExpressionEvaluatorType, LogLevel, WorkflowSettings } from 'n8n-workflow'; export interface IVersionNotificationSettings { @@ -169,4 +170,5 @@ export interface FrontendSettings { security: { blockFileAccessToN8nFiles: boolean; }; + betaFeatures: FrontendBetaFeatures[]; } diff --git a/packages/@n8n/benchmark/README.md b/packages/@n8n/benchmark/README.md index b3a0a4d548..af8726d0ea 100644 --- a/packages/@n8n/benchmark/README.md +++ b/packages/@n8n/benchmark/README.md @@ -1,6 +1,6 @@ # n8n benchmarking tool -Tool for executing benchmarks against an n8n instance. The tool consists of these components: +Tool for executing benchmarks against an n8n instance. ## Directory structure @@ -12,6 +12,39 @@ packages/@n8n/benchmark ├── scripts Orchestration scripts ``` +## Benchmarking an existing n8n instance + +The easiest way to run the existing benchmark scenarios is to use the benchmark docker image: + +```sh +docker pull ghcr.io/n8n-io/n8n-benchmark:latest +# Print the help to list all available flags +docker run ghcr.io/n8n-io/n8n-benchmark:latest run --help +# Run all available benchmark scenarios for 1 minute with 5 concurrent requests +docker run ghcr.io/n8n-io/n8n-benchmark:latest run \ + --n8nBaseUrl=https://instance.url \ + --n8nUserEmail=InstanceOwner@email.com \ + --n8nUserPassword=InstanceOwnerPassword \ + --vus=5 \ + --duration=1m \ + --scenarioFilter SingleWebhook +``` + +### Using custom scenarios with the Docker image + +It is also possible to create your own [benchmark scenarios](#benchmark-scenarios) and load them using the `--testScenariosPath` flag: + +```sh +# Assuming your scenarios are located in `./scenarios`, mount them into `/scenarios` in the container +docker run -v ./scenarios:/scenarios ghcr.io/n8n-io/n8n-benchmark:latest run \ + --n8nBaseUrl=https://instance.url \ + --n8nUserEmail=InstanceOwner@email.com \ + --n8nUserPassword=InstanceOwnerPassword \ + --vus=5 \ + --duration=1m \ + --testScenariosPath=/scenarios +``` + ## Running the entire benchmark suite The benchmark suite consists of [benchmark scenarios](#benchmark-scenarios) and different [n8n setups](#n8n-setups). diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json index 98edd1dabd..abb8514c3d 100644 --- a/packages/@n8n/benchmark/package.json +++ b/packages/@n8n/benchmark/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/n8n-benchmark", - "version": "1.7.0", + "version": "1.8.0", "description": "Cli for running benchmark tests for n8n", "main": "dist/index", "scripts": { @@ -17,7 +17,7 @@ "benchmark-locally": "pnpm benchmark --env local", "provision-cloud-env": "zx scripts/provision-cloud-env.mjs", "destroy-cloud-env": "zx scripts/destroy-cloud-env.mjs", - "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" + "watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\"" }, "engines": { "node": ">=20.10" @@ -41,10 +41,7 @@ }, "devDependencies": { "@types/convict": "^6.1.1", - "@types/k6": "^0.52.0", - "@types/node": "^20.14.8", - "tsc-alias": "^1.8.7", - "typescript": "^5.5.2" + "@types/k6": "^0.52.0" }, "bin": { "n8n-benchmark": "./bin/n8n-benchmark" diff --git a/packages/@n8n/benchmark/scenarios/binary-data/binary-data.script.js b/packages/@n8n/benchmark/scenarios/binary-data/binary-data.script.js index c10e667cbb..28edfdf9ec 100644 --- a/packages/@n8n/benchmark/scenarios/binary-data/binary-data.script.js +++ b/packages/@n8n/benchmark/scenarios/binary-data/binary-data.script.js @@ -15,6 +15,12 @@ export default function () { const res = http.post(`${apiBaseUrl}/webhook/binary-files-benchmark`, data); + if (res.status !== 200) { + console.error( + `Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`, + ); + } + check(res, { 'is status 200': (r) => r.status === 200, 'has correct content type': (r) => diff --git a/packages/@n8n/benchmark/scenarios/http-node/http-node.script.js b/packages/@n8n/benchmark/scenarios/http-node/http-node.script.js index b391982259..4ecee9d1bd 100644 --- a/packages/@n8n/benchmark/scenarios/http-node/http-node.script.js +++ b/packages/@n8n/benchmark/scenarios/http-node/http-node.script.js @@ -6,6 +6,12 @@ const apiBaseUrl = __ENV.API_BASE_URL; export default function () { const res = http.post(`${apiBaseUrl}/webhook/benchmark-http-node`); + if (res.status !== 200) { + console.error( + `Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`, + ); + } + check(res, { 'is status 200': (r) => r.status === 200, 'http requests were OK': (r) => { diff --git a/packages/@n8n/benchmark/scenarios/js-code-node/js-code-node.script.js b/packages/@n8n/benchmark/scenarios/js-code-node/js-code-node.script.js index 74cef4f441..b2fd8eb315 100644 --- a/packages/@n8n/benchmark/scenarios/js-code-node/js-code-node.script.js +++ b/packages/@n8n/benchmark/scenarios/js-code-node/js-code-node.script.js @@ -5,6 +5,13 @@ const apiBaseUrl = __ENV.API_BASE_URL; export default function () { const res = http.post(`${apiBaseUrl}/webhook/code-node-benchmark`, {}); + + if (res.status !== 200) { + console.error( + `Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`, + ); + } + check(res, { 'is status 200': (r) => r.status === 200, 'has items in response': (r) => { diff --git a/packages/@n8n/benchmark/scenarios/set-node-expressions/set-node-expressions.script.js b/packages/@n8n/benchmark/scenarios/set-node-expressions/set-node-expressions.script.js index 4bea17eb9f..9564fcc53c 100644 --- a/packages/@n8n/benchmark/scenarios/set-node-expressions/set-node-expressions.script.js +++ b/packages/@n8n/benchmark/scenarios/set-node-expressions/set-node-expressions.script.js @@ -5,6 +5,13 @@ const apiBaseUrl = __ENV.API_BASE_URL; export default function () { const res = http.post(`${apiBaseUrl}/webhook/set-expressions-benchmark`, {}); + + if (res.status !== 200) { + console.error( + `Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`, + ); + } + check(res, { 'is status 200': (r) => r.status === 200, }); diff --git a/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.manifest.json b/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.manifest.json index 9c68908eef..2113c73ec9 100644 --- a/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.manifest.json +++ b/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.manifest.json @@ -3,5 +3,5 @@ "name": "SingleWebhook", "description": "A single webhook trigger that responds with a 200 status code", "scenarioData": { "workflowFiles": ["single-webhook.json"] }, - "scriptPath": "single-webhook.script.ts" + "scriptPath": "single-webhook.script.js" } diff --git a/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.ts b/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.js similarity index 63% rename from packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.ts rename to packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.js index 72e2563cbe..41facc8aeb 100644 --- a/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.ts +++ b/packages/@n8n/benchmark/scenarios/single-webhook/single-webhook.script.js @@ -5,6 +5,13 @@ const apiBaseUrl = __ENV.API_BASE_URL; export default function () { const res = http.get(`${apiBaseUrl}/webhook/single-webhook`); + + if (res.status !== 200) { + console.error( + `Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`, + ); + } + check(res, { 'is status 200': (r) => r.status === 200, }); diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml index ca3ad9c23d..c686f581b3 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml @@ -176,7 +176,7 @@ services: # Load balancer that acts as an entry point for n8n n8n: - image: nginx:latest + image: nginx:1.27.2 ports: - '5678:80' volumes: diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/nginx.conf b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/nginx.conf index 86100f8c50..142da7416e 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/nginx.conf +++ b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/nginx.conf @@ -3,6 +3,7 @@ events {} http { client_max_body_size 50M; access_log off; + error_log /dev/stderr warn; upstream backend { server n8n_main1:5678; diff --git a/packages/@n8n/benchmark/scripts/run-in-cloud.mjs b/packages/@n8n/benchmark/scripts/run-in-cloud.mjs index c61c0901d4..35e90bdee5 100755 --- a/packages/@n8n/benchmark/scripts/run-in-cloud.mjs +++ b/packages/@n8n/benchmark/scripts/run-in-cloud.mjs @@ -78,12 +78,6 @@ async function runBenchmarksOnVm(config, benchmarkEnv) { const bootstrapScriptPath = path.join(scriptsDir, 'bootstrap.sh'); await sshClient.ssh(`chmod a+x ${bootstrapScriptPath} && ${bootstrapScriptPath}`); - // Benchmarking the VM - const vmBenchmarkScriptPath = path.join(scriptsDir, 'vm-benchmark.sh'); - await sshClient.ssh(`chmod a+x ${vmBenchmarkScriptPath} && ${vmBenchmarkScriptPath}`, { - verbose: true, - }); - // Give some time for the VM to be ready await sleep(1000); diff --git a/packages/@n8n/benchmark/scripts/vm-benchmark.sh b/packages/@n8n/benchmark/scripts/vm-benchmark.sh deleted file mode 100644 index 13b7eb2b1a..0000000000 --- a/packages/@n8n/benchmark/scripts/vm-benchmark.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Install fio -DEBIAN_FRONTEND=noninteractive sudo apt-get -y install fio > /dev/null - -# Run the disk benchmark -fio --name=rand_rw --ioengine=libaio --rw=randrw --rwmixread=70 --bs=4k --numjobs=4 --size=1G --runtime=30 --directory=/n8n --group_reporting - -# Remove files -sudo rm /n8n/rand_rw.* - -# Uninstall fio -DEBIAN_FRONTEND=noninteractive sudo apt-get -y remove fio > /dev/null diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts index 164eef0f41..312de91c3c 100644 --- a/packages/@n8n/benchmark/src/commands/run.ts +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -13,6 +13,10 @@ export default class RunCommand extends Command { static flags = { testScenariosPath, + scenarioFilter: Flags.string({ + char: 'f', + description: 'Filter scenarios by name', + }), scenarioNamePrefix: Flags.string({ description: 'Prefix for the scenario name', default: 'Unnamed', @@ -95,7 +99,7 @@ export default class RunCommand extends Command { flags.scenarioNamePrefix, ); - const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath); + const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath, flags.scenarioFilter); await scenarioRunner.runManyScenarios(allScenarios); } diff --git a/packages/@n8n/benchmark/src/scenario/scenario-loader.ts b/packages/@n8n/benchmark/src/scenario/scenario-loader.ts index 4f315c1bf7..2da9109e86 100644 --- a/packages/@n8n/benchmark/src/scenario/scenario-loader.ts +++ b/packages/@n8n/benchmark/src/scenario/scenario-loader.ts @@ -8,7 +8,7 @@ export class ScenarioLoader { /** * Loads all scenarios from the given path */ - loadAll(pathToScenarios: string): Scenario[] { + loadAll(pathToScenarios: string, filter?: string): Scenario[] { pathToScenarios = path.resolve(pathToScenarios); const scenarioFolders = fs .readdirSync(pathToScenarios, { withFileTypes: true }) @@ -18,6 +18,9 @@ export class ScenarioLoader { const scenarios: Scenario[] = []; for (const folder of scenarioFolders) { + if (filter && !folder.toLowerCase().includes(filter.toLowerCase())) { + continue; + } const scenarioPath = path.join(pathToScenarios, folder); const manifestFileName = `${folder}.manifest.json`; const scenarioManifestPath = path.join(pathToScenarios, folder, manifestFileName); diff --git a/packages/@n8n/benchmark/src/test-execution/scenario-runner.ts b/packages/@n8n/benchmark/src/test-execution/scenario-runner.ts index 84d1d8b096..def841ccf5 100644 --- a/packages/@n8n/benchmark/src/test-execution/scenario-runner.ts +++ b/packages/@n8n/benchmark/src/test-execution/scenario-runner.ts @@ -1,3 +1,5 @@ +import { sleep } from 'zx'; + import { AuthenticatedN8nApiClient } from '@/n8n-api-client/authenticated-n8n-api-client'; import type { N8nApiClient } from '@/n8n-api-client/n8n-api-client'; import type { ScenarioDataFileLoader } from '@/scenario/scenario-data-loader'; @@ -47,6 +49,10 @@ export class ScenarioRunner { const testData = await this.dataLoader.loadDataForScenario(scenario); await testDataImporter.importTestScenarioData(testData.workflows); + // Wait for 1s before executing the scenario to ensure that the workflows are activated. + // In multi-main mode it can take some time before the workflow becomes active. + await sleep(1000); + console.log('Executing scenario script'); await this.k6Executor.executeTestScenario(scenario, { scenarioRunName, diff --git a/packages/@n8n/chat/package.json b/packages/@n8n/chat/package.json index 24d6cf6f1c..5dc5881181 100644 --- a/packages/@n8n/chat/package.json +++ b/packages/@n8n/chat/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/chat", - "version": "0.28.0", + "version": "0.29.0", "scripts": { "dev": "pnpm run storybook", "build": "pnpm build:vite && pnpm build:bundle", diff --git a/packages/@n8n/config/package.json b/packages/@n8n/config/package.json index 10c8cbcf5b..ad583c1108 100644 --- a/packages/@n8n/config/package.json +++ b/packages/@n8n/config/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/config", - "version": "1.13.0", + "version": "1.15.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/config/src/configs/frontend.config.ts b/packages/@n8n/config/src/configs/frontend.config.ts new file mode 100644 index 0000000000..63f812952f --- /dev/null +++ b/packages/@n8n/config/src/configs/frontend.config.ts @@ -0,0 +1,11 @@ +import { Config, Env } from '../decorators'; +import { StringArray } from '../utils'; + +export type FrontendBetaFeatures = 'canvas_v2'; + +@Config +export class FrontendConfig { + /** Which UI experiments to enable. Separate multiple values with a comma `,` */ + @Env('N8N_UI_BETA_FEATURES') + betaFeatures: StringArray = []; +} diff --git a/packages/@n8n/config/src/configs/generic.config.ts b/packages/@n8n/config/src/configs/generic.config.ts new file mode 100644 index 0000000000..f6960b2415 --- /dev/null +++ b/packages/@n8n/config/src/configs/generic.config.ts @@ -0,0 +1,15 @@ +import { Config, Env } from '../decorators'; + +@Config +export class GenericConfig { + /** Default timezone for the n8n instance. Can be overridden on a per-workflow basis. */ + @Env('GENERIC_TIMEZONE') + timezone: string = 'America/New_York'; + + @Env('N8N_RELEASE_TYPE') + releaseChannel: 'stable' | 'beta' | 'nightly' | 'dev' = 'dev'; + + /** Grace period (in seconds) to wait for components to shut down before process exit. */ + @Env('N8N_GRACEFUL_SHUTDOWN_TIMEOUT') + gracefulShutdownTimeout: number = 30; +} diff --git a/packages/@n8n/config/src/configs/license.config.ts b/packages/@n8n/config/src/configs/license.config.ts new file mode 100644 index 0000000000..58ccef450c --- /dev/null +++ b/packages/@n8n/config/src/configs/license.config.ts @@ -0,0 +1,28 @@ +import { Config, Env } from '../decorators'; + +@Config +export class LicenseConfig { + /** License server URL to retrieve license. */ + @Env('N8N_LICENSE_SERVER_URL') + serverUrl: string = 'https://license.n8n.io/v1'; + + /** Whether autorenewal for licenses is enabled. */ + @Env('N8N_LICENSE_AUTO_RENEW_ENABLED') + autoRenewalEnabled: boolean = true; + + /** How long (in seconds) before expiry a license should be autorenewed. */ + @Env('N8N_LICENSE_AUTO_RENEW_OFFSET') + autoRenewOffset: number = 60 * 60 * 72; // 72 hours + + /** Activation key to initialize license. */ + @Env('N8N_LICENSE_ACTIVATION_KEY') + activationKey: string = ''; + + /** Tenant ID used by the license manager SDK, e.g. for self-hosted, sandbox, embed, cloud. */ + @Env('N8N_LICENSE_TENANT_ID') + tenantId: number = 1; + + /** Ephemeral license certificate. See: https://github.com/n8n-io/license-management?tab=readme-ov-file#concept-ephemeral-entitlements */ + @Env('N8N_LICENSE_CERT') + cert: string = ''; +} diff --git a/packages/@n8n/config/src/configs/logging.config.ts b/packages/@n8n/config/src/configs/logging.config.ts index c29bb6d5a8..94e4642223 100644 --- a/packages/@n8n/config/src/configs/logging.config.ts +++ b/packages/@n8n/config/src/configs/logging.config.ts @@ -1,14 +1,18 @@ import { Config, Env, Nested } from '../decorators'; import { StringArray } from '../utils'; -/** - * Scopes (areas of functionality) to filter logs by. - * - * `executions` -> execution lifecycle - * `license` -> license SDK - * `scaling` -> scaling mode - */ -export const LOG_SCOPES = ['executions', 'license', 'scaling'] as const; +/** Scopes (areas of functionality) to filter logs by. */ +export const LOG_SCOPES = [ + 'concurrency', + 'external-secrets', + 'license', + 'multi-main-setup', + 'pubsub', + 'redis', + 'scaling', + 'waiting-executions', + 'task-runner', +] as const; export type LogScope = (typeof LOG_SCOPES)[number]; @@ -59,14 +63,20 @@ export class LoggingConfig { /** * Scopes to filter logs by. Nothing is filtered by default. * - * Currently supported log scopes: - * - `executions` + * Supported log scopes: + * + * - `concurrency` + * - `external-secrets` * - `license` + * - `multi-main-setup` + * - `pubsub` + * - `redis` * - `scaling` + * - `waiting-executions` * * @example * `N8N_LOG_SCOPES=license` - * `N8N_LOG_SCOPES=license,executions` + * `N8N_LOG_SCOPES=license,waiting-executions` */ @Env('N8N_LOG_SCOPES') scopes: StringArray = []; diff --git a/packages/@n8n/config/src/configs/multi-main-setup.config.ts b/packages/@n8n/config/src/configs/multi-main-setup.config.ts new file mode 100644 index 0000000000..e3599c1d55 --- /dev/null +++ b/packages/@n8n/config/src/configs/multi-main-setup.config.ts @@ -0,0 +1,16 @@ +import { Config, Env } from '../decorators'; + +@Config +export class MultiMainSetupConfig { + /** Whether to enable multi-main setup (if licensed) for scaling mode. */ + @Env('N8N_MULTI_MAIN_SETUP_ENABLED') + enabled: boolean = false; + + /** Time to live (in seconds) for leader key in multi-main setup. */ + @Env('N8N_MULTI_MAIN_SETUP_KEY_TTL') + ttl: number = 10; + + /** Interval (in seconds) for leader check in multi-main setup. */ + @Env('N8N_MULTI_MAIN_SETUP_CHECK_INTERVAL') + interval: number = 3; +} diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts index e7335e8827..c7be197963 100644 --- a/packages/@n8n/config/src/configs/runners.config.ts +++ b/packages/@n8n/config/src/configs/runners.config.ts @@ -1,11 +1,23 @@ import { Config, Env } from '../decorators'; +/** + * Whether to enable task runners and how to run them + * - internal_childprocess: Task runners are run as a child process and launched by n8n + * - internal_launcher: Task runners are run as a child process and launched by n8n using a separate launch program + * - external: Task runners are run as a separate program not launched by n8n + */ +export type TaskRunnerMode = 'internal_childprocess' | 'internal_launcher' | 'external'; + @Config export class TaskRunnersConfig { // Defaults to true for now @Env('N8N_RUNNERS_DISABLED') disabled: boolean = true; + // Defaults to true for now + @Env('N8N_RUNNERS_MODE') + mode: TaskRunnerMode = 'internal_childprocess'; + @Env('N8N_RUNNERS_PATH') path: string = '/runners'; @@ -18,5 +30,24 @@ export class TaskRunnersConfig { /** IP address task runners server should listen on */ @Env('N8N_RUNNERS_SERVER_LISTEN_ADDRESS') - listen_address: string = '127.0.0.1'; + listenAddress: string = '127.0.0.1'; + + /** Maximum size of a payload sent to the runner in bytes, Default 1G */ + @Env('N8N_RUNNERS_MAX_PAYLOAD') + maxPayload: number = 1024 * 1024 * 1024; + + @Env('N8N_RUNNERS_LAUNCHER_PATH') + launcherPath: string = ''; + + /** Which task runner to launch from the config */ + @Env('N8N_RUNNERS_LAUNCHER_RUNNER') + launcherRunner: string = 'javascript'; + + /** The --max-old-space-size option to use for the runner (in MB). Default means node.js will determine it based on the available memory. */ + @Env('N8N_RUNNERS_MAX_OLD_SPACE_SIZE') + maxOldSpaceSize: string = ''; + + /** How many concurrent tasks can a runner execute at a time */ + @Env('N8N_RUNNERS_MAX_CONCURRENCY') + maxConcurrency: number = 5; } diff --git a/packages/@n8n/config/src/configs/scaling-mode.config.ts b/packages/@n8n/config/src/configs/scaling-mode.config.ts index 05ee6b4841..f202440a5b 100644 --- a/packages/@n8n/config/src/configs/scaling-mode.config.ts +++ b/packages/@n8n/config/src/configs/scaling-mode.config.ts @@ -82,10 +82,6 @@ class BullConfig { @Nested redis: RedisConfig; - /** How often (in seconds) to poll the Bull queue to identify executions finished during a Redis crash. `0` to disable. May increase Redis traffic significantly. */ - @Env('QUEUE_RECOVERY_INTERVAL') - queueRecoveryInterval: number = 60; // watchdog interval - /** @deprecated How long (in seconds) a worker must wait for active executions to finish before exiting. Use `N8N_GRACEFUL_SHUTDOWN_TIMEOUT` instead */ @Env('QUEUE_WORKER_TIMEOUT') gracefulShutdownTimeout: number = 30; diff --git a/packages/@n8n/config/src/configs/security.config.ts b/packages/@n8n/config/src/configs/security.config.ts new file mode 100644 index 0000000000..329e84cc43 --- /dev/null +++ b/packages/@n8n/config/src/configs/security.config.ts @@ -0,0 +1,27 @@ +import { Config, Env } from '../decorators'; + +@Config +export class SecurityConfig { + /** + * Which directories to limit n8n's access to. Separate multiple dirs with semicolon `;`. + * + * @example N8N_RESTRICT_FILE_ACCESS_TO=/home/user/.n8n;/home/user/n8n-data + */ + @Env('N8N_RESTRICT_FILE_ACCESS_TO') + restrictFileAccessTo: string = ''; + + /** + * Whether to block access to all files at: + * - the ".n8n" directory, + * - the static cache dir at ~/.cache/n8n/public, and + * - user-defined config files. + */ + @Env('N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES') + blockFileAccessToN8nFiles: boolean = true; + + /** + * In a [security audit](https://docs.n8n.io/hosting/securing/security-audit/), how many days for a workflow to be considered abandoned if not executed. + */ + @Env('N8N_SECURITY_AUDIT_DAYS_ABANDONED_WORKFLOW') + daysAbandonedWorkflow: number = 90; +} diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index 9044ffa0fa..c056a1090c 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -5,12 +5,15 @@ import { EndpointsConfig } from './configs/endpoints.config'; import { EventBusConfig } from './configs/event-bus.config'; import { ExternalSecretsConfig } from './configs/external-secrets.config'; import { ExternalStorageConfig } from './configs/external-storage.config'; +import { GenericConfig } from './configs/generic.config'; +import { LicenseConfig } from './configs/license.config'; import { LoggingConfig } from './configs/logging.config'; +import { MultiMainSetupConfig } from './configs/multi-main-setup.config'; import { NodesConfig } from './configs/nodes.config'; import { PublicApiConfig } from './configs/public-api.config'; import { TaskRunnersConfig } from './configs/runners.config'; -export { TaskRunnersConfig } from './configs/runners.config'; import { ScalingModeConfig } from './configs/scaling-mode.config'; +import { SecurityConfig } from './configs/security.config'; import { SentryConfig } from './configs/sentry.config'; import { TemplatesConfig } from './configs/templates.config'; import { UserManagementConfig } from './configs/user-management.config'; @@ -18,6 +21,10 @@ import { VersionNotificationsConfig } from './configs/version-notifications.conf import { WorkflowsConfig } from './configs/workflows.config'; import { Config, Env, Nested } from './decorators'; +export { Config, Env, Nested } from './decorators'; +export { TaskRunnersConfig } from './configs/runners.config'; +export { SecurityConfig } from './configs/security.config'; +export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config'; export { LOG_SCOPES } from './configs/logging.config'; export type { LogScope } from './configs/logging.config'; @@ -93,4 +100,16 @@ export class GlobalConfig { @Nested taskRunners: TaskRunnersConfig; + + @Nested + multiMainSetup: MultiMainSetupConfig; + + @Nested + generic: GenericConfig; + + @Nested + license: LicenseConfig; + + @Nested + security: SecurityConfig; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 301022ca3e..07af2c0a0b 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -211,7 +211,6 @@ describe('GlobalConfig', () => { clusterNodes: '', tls: false, }, - queueRecoveryInterval: 60, gracefulShutdownTimeout: 30, prefix: 'bull', settings: { @@ -224,10 +223,16 @@ describe('GlobalConfig', () => { }, taskRunners: { disabled: true, + mode: 'internal_childprocess', path: '/runners', authToken: '', - listen_address: '127.0.0.1', + listenAddress: '127.0.0.1', + maxPayload: 1024 * 1024 * 1024, port: 5679, + launcherPath: '', + launcherRunner: 'javascript', + maxOldSpaceSize: '', + maxConcurrency: 5, }, sentry: { backendDsn: '', @@ -243,6 +248,29 @@ describe('GlobalConfig', () => { }, scopes: [], }, + multiMainSetup: { + enabled: false, + ttl: 10, + interval: 3, + }, + generic: { + timezone: 'America/New_York', + releaseChannel: 'dev', + gracefulShutdownTimeout: 30, + }, + license: { + serverUrl: 'https://license.n8n.io/v1', + autoRenewalEnabled: true, + autoRenewOffset: 60 * 60 * 72, + activationKey: '', + tenantId: 1, + cert: '', + }, + security: { + restrictFileAccessTo: '', + blockFileAccessToN8nFiles: true, + daysAbandonedWorkflow: 90, + }, }; it('should use all default values when no env variables are defined', () => { diff --git a/packages/@n8n/json-schema-to-zod/.eslintrc.js b/packages/@n8n/json-schema-to-zod/.eslintrc.js new file mode 100644 index 0000000000..03caaf4930 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/.eslintrc.js @@ -0,0 +1,21 @@ +const sharedOptions = require('@n8n_io/eslint-config/shared'); + +/** + * @type {import('@types/eslint').ESLint.ConfigData} + */ +module.exports = { + extends: ['@n8n_io/eslint-config/node'], + + ...sharedOptions(__dirname), + + ignorePatterns: ['jest.config.js'], + + rules: { + 'unicorn/filename-case': ['error', { case: 'kebabCase' }], + '@typescript-eslint/no-duplicate-imports': 'off', + 'import/no-cycle': 'off', + 'n8n-local-rules/no-plain-errors': 'off', + + complexity: 'error', + }, +}; diff --git a/packages/@n8n/json-schema-to-zod/.gitignore b/packages/@n8n/json-schema-to-zod/.gitignore new file mode 100644 index 0000000000..d11ff827d5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +coverage +test/output diff --git a/packages/@n8n/json-schema-to-zod/.npmignore b/packages/@n8n/json-schema-to-zod/.npmignore new file mode 100644 index 0000000000..3aeebeb66b --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/.npmignore @@ -0,0 +1,3 @@ +src +tsconfig* +test diff --git a/packages/@n8n/json-schema-to-zod/LICENSE b/packages/@n8n/json-schema-to-zod/LICENSE new file mode 100644 index 0000000000..aa24f46da6 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/LICENSE @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) 2024, n8n +Copyright (c) 2021, Stefan Terdell + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/packages/@n8n/json-schema-to-zod/README.md b/packages/@n8n/json-schema-to-zod/README.md new file mode 100644 index 0000000000..cb76a141b5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/README.md @@ -0,0 +1,34 @@ +# Json-Schema-to-Zod + +A package to convert JSON schema (draft 4+) objects into Zod schemas in the form of Zod objects at runtime. + +## Installation + +```sh +npm install @n8n/json-schema-to-zod +``` + +### Simple example + +```typescript +import { jsonSchemaToZod } from "json-schema-to-zod"; + +const jsonSchema = { + type: "object", + properties: { + hello: { + type: "string", + }, + }, +}; + +const zodSchema = jsonSchemaToZod(myObject); +``` + +### Overriding a parser + +You can pass a function to the `overrideParser` option, which represents a function that receives the current schema node and the reference object, and should return a zod object when it wants to replace a default output. If the default output should be used for the node just return undefined. + +## Acknowledgements + +This is a fork of [`json-schema-to-zod`](https://github.com/StefanTerdell/json-schema-to-zod). diff --git a/packages/@n8n/json-schema-to-zod/jest.config.js b/packages/@n8n/json-schema-to-zod/jest.config.js new file mode 100644 index 0000000000..b8e98e8970 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/jest.config.js @@ -0,0 +1,5 @@ +/** @type {import('jest').Config} */ +module.exports = { + ...require('../../../jest.config'), + setupFilesAfterEnv: ['/test/extend-expect.ts'], +}; diff --git a/packages/@n8n/json-schema-to-zod/package.json b/packages/@n8n/json-schema-to-zod/package.json new file mode 100644 index 0000000000..e1ae6beaa9 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/package.json @@ -0,0 +1,68 @@ +{ + "name": "@n8n/json-schema-to-zod", + "version": "1.1.0", + "description": "Converts JSON schema objects into Zod schemas", + "types": "./dist/types/index.d.ts", + "main": "./dist/cjs/index.js", + "module": "./dist/esm/index.js", + "exports": { + "import": { + "types": "./dist/types/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/types/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "scripts": { + "clean": "rimraf dist .turbo", + "typecheck": "tsc --noEmit", + "dev": "tsc -w", + "format": "biome format --write src", + "format:check": "biome ci src", + "lint": "eslint . --quiet", + "lintfix": "eslint . --fix", + "build:types": "tsc -p tsconfig.types.json", + "build:cjs": "tsc -p tsconfig.cjs.json && node postcjs.js", + "build:esm": "tsc -p tsconfig.esm.json && node postesm.js", + "build": "rimraf ./dist && pnpm run build:types && pnpm run build:cjs && pnpm run build:esm", + "dry": "pnpm run build && pnpm pub --dry-run", + "test": "jest", + "test:watch": "jest --watch" + }, + "keywords": [ + "zod", + "json", + "schema", + "converter", + "cli" + ], + "author": "Stefan Terdell", + "contributors": [ + "Chen (https://github.com/werifu)", + "Nuno Carduso (https://github.com/ncardoso-barracuda)", + "Lars Strojny (https://github.com/lstrojny)", + "Navtoj Chahal (https://github.com/navtoj)", + "Ben McCann (https://github.com/benmccann)", + "Dmitry Zakharov (https://github.com/DZakh)", + "Michel Turpin (https://github.com/grimly)", + "David Barratt (https://github.com/davidbarratt)", + "pevisscher (https://github.com/pevisscher)", + "Aidin Abedi (https://github.com/aidinabedi)", + "Brett Zamir (https://github.com/brettz9)", + "n8n (https://github.com/n8n-io)" + ], + "license": "ISC", + "repository": { + "type": "git", + "url": "https://github.com/n8n-io/n8n" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "devDependencies": { + "@types/json-schema": "^7.0.15", + "zod": "catalog:" + } +} diff --git a/packages/@n8n/json-schema-to-zod/postcjs.js b/packages/@n8n/json-schema-to-zod/postcjs.js new file mode 100644 index 0000000000..618aa03a96 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/postcjs.js @@ -0,0 +1 @@ +require('fs').writeFileSync('./dist/cjs/package.json', '{"type":"commonjs"}', 'utf-8'); diff --git a/packages/@n8n/json-schema-to-zod/postesm.js b/packages/@n8n/json-schema-to-zod/postesm.js new file mode 100644 index 0000000000..5235734d6c --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/postesm.js @@ -0,0 +1 @@ +require('fs').writeFileSync('./dist/esm/package.json', '{"type":"module"}', 'utf-8'); diff --git a/packages/@n8n/json-schema-to-zod/src/index.ts b/packages/@n8n/json-schema-to-zod/src/index.ts new file mode 100644 index 0000000000..10dae97784 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/index.ts @@ -0,0 +1,2 @@ +export type * from './types'; +export { jsonSchemaToZod } from './json-schema-to-zod.js'; diff --git a/packages/@n8n/json-schema-to-zod/src/json-schema-to-zod.ts b/packages/@n8n/json-schema-to-zod/src/json-schema-to-zod.ts new file mode 100644 index 0000000000..6f1c6a1315 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/json-schema-to-zod.ts @@ -0,0 +1,15 @@ +import type { z } from 'zod'; + +import { parseSchema } from './parsers/parse-schema'; +import type { JsonSchemaToZodOptions, JsonSchema } from './types'; + +export const jsonSchemaToZod = ( + schema: JsonSchema, + options: JsonSchemaToZodOptions = {}, +): T => { + return parseSchema(schema, { + path: [], + seen: new Map(), + ...options, + }) as T; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-all-of.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-all-of.ts new file mode 100644 index 0000000000..be8fd2c7e5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-all-of.ts @@ -0,0 +1,46 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, JsonSchema, Refs } from '../types'; +import { half } from '../utils/half'; + +const originalIndex = Symbol('Original index'); + +const ensureOriginalIndex = (arr: JsonSchema[]) => { + const newArr = []; + + for (let i = 0; i < arr.length; i++) { + const item = arr[i]; + if (typeof item === 'boolean') { + newArr.push(item ? { [originalIndex]: i } : { [originalIndex]: i, not: {} }); + } else if (originalIndex in item) { + return arr; + } else { + newArr.push({ ...item, [originalIndex]: i }); + } + } + + return newArr; +}; + +export function parseAllOf( + jsonSchema: JsonSchemaObject & { allOf: JsonSchema[] }, + refs: Refs, +): z.ZodTypeAny { + if (jsonSchema.allOf.length === 0) { + return z.never(); + } + + if (jsonSchema.allOf.length === 1) { + const item = jsonSchema.allOf[0]; + + return parseSchema(item, { + ...refs, + path: [...refs.path, 'allOf', (item as never)[originalIndex]], + }); + } + + const [left, right] = half(ensureOriginalIndex(jsonSchema.allOf)); + + return z.intersection(parseAllOf({ allOf: left }, refs), parseAllOf({ allOf: right }, refs)); +} diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-any-of.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-any-of.ts new file mode 100644 index 0000000000..73b19b1739 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-any-of.ts @@ -0,0 +1,19 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, JsonSchema, Refs } from '../types'; + +export const parseAnyOf = (jsonSchema: JsonSchemaObject & { anyOf: JsonSchema[] }, refs: Refs) => { + return jsonSchema.anyOf.length + ? jsonSchema.anyOf.length === 1 + ? parseSchema(jsonSchema.anyOf[0], { + ...refs, + path: [...refs.path, 'anyOf', 0], + }) + : z.union( + jsonSchema.anyOf.map((schema, i) => + parseSchema(schema, { ...refs, path: [...refs.path, 'anyOf', i] }), + ) as [z.ZodTypeAny, z.ZodTypeAny], + ) + : z.any(); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-array.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-array.ts new file mode 100644 index 0000000000..5e01473fd6 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-array.ts @@ -0,0 +1,34 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, Refs } from '../types'; +import { extendSchemaWithMessage } from '../utils/extend-schema'; + +export const parseArray = (jsonSchema: JsonSchemaObject & { type: 'array' }, refs: Refs) => { + if (Array.isArray(jsonSchema.items)) { + return z.tuple( + jsonSchema.items.map((v, i) => + parseSchema(v, { ...refs, path: [...refs.path, 'items', i] }), + ) as [z.ZodTypeAny], + ); + } + + let zodSchema = !jsonSchema.items + ? z.array(z.any()) + : z.array(parseSchema(jsonSchema.items, { ...refs, path: [...refs.path, 'items'] })); + + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'minItems', + (zs, minItems, errorMessage) => zs.min(minItems, errorMessage), + ); + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'maxItems', + (zs, maxItems, errorMessage) => zs.max(maxItems, errorMessage), + ); + + return zodSchema; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-boolean.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-boolean.ts new file mode 100644 index 0000000000..be8e309e43 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-boolean.ts @@ -0,0 +1,7 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; + +export const parseBoolean = (_jsonSchema: JsonSchemaObject & { type: 'boolean' }) => { + return z.boolean(); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-const.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-const.ts new file mode 100644 index 0000000000..445523652d --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-const.ts @@ -0,0 +1,7 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject, Serializable } from '../types'; + +export const parseConst = (jsonSchema: JsonSchemaObject & { const: Serializable }) => { + return z.literal(jsonSchema.const as z.Primitive); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-default.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-default.ts new file mode 100644 index 0000000000..d64bcf85c8 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-default.ts @@ -0,0 +1,7 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; + +export const parseDefault = (_jsonSchema: JsonSchemaObject) => { + return z.any(); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-enum.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-enum.ts new file mode 100644 index 0000000000..26385472cc --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-enum.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject, Serializable } from '../types'; + +export const parseEnum = (jsonSchema: JsonSchemaObject & { enum: Serializable[] }) => { + if (jsonSchema.enum.length === 0) { + return z.never(); + } + + if (jsonSchema.enum.length === 1) { + // union does not work when there is only one element + return z.literal(jsonSchema.enum[0] as z.Primitive); + } + + if (jsonSchema.enum.every((x) => typeof x === 'string')) { + return z.enum(jsonSchema.enum as [string]); + } + + return z.union( + jsonSchema.enum.map((x) => z.literal(x as z.Primitive)) as unknown as [ + z.ZodTypeAny, + z.ZodTypeAny, + ], + ); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-if-then-else.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-if-then-else.ts new file mode 100644 index 0000000000..7cb595a615 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-if-then-else.ts @@ -0,0 +1,31 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, JsonSchema, Refs } from '../types'; + +export const parseIfThenElse = ( + jsonSchema: JsonSchemaObject & { + if: JsonSchema; + then: JsonSchema; + else: JsonSchema; + }, + refs: Refs, +) => { + const $if = parseSchema(jsonSchema.if, { ...refs, path: [...refs.path, 'if'] }); + const $then = parseSchema(jsonSchema.then, { + ...refs, + path: [...refs.path, 'then'], + }); + const $else = parseSchema(jsonSchema.else, { + ...refs, + path: [...refs.path, 'else'], + }); + + return z.union([$then, $else]).superRefine((value, ctx) => { + const result = $if.safeParse(value).success ? $then.safeParse(value) : $else.safeParse(value); + + if (!result.success) { + result.error.errors.forEach((error) => ctx.addIssue(error)); + } + }); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-multiple-type.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-multiple-type.ts new file mode 100644 index 0000000000..65ff3c35b5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-multiple-type.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchema, JsonSchemaObject, Refs } from '../types'; + +export const parseMultipleType = ( + jsonSchema: JsonSchemaObject & { type: string[] }, + refs: Refs, +) => { + return z.union( + jsonSchema.type.map((type) => parseSchema({ ...jsonSchema, type } as JsonSchema, refs)) as [ + z.ZodTypeAny, + z.ZodTypeAny, + ], + ); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-not.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-not.ts new file mode 100644 index 0000000000..219d32c8dd --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-not.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, JsonSchema, Refs } from '../types'; + +export const parseNot = (jsonSchema: JsonSchemaObject & { not: JsonSchema }, refs: Refs) => { + return z.any().refine( + (value) => + !parseSchema(jsonSchema.not, { + ...refs, + path: [...refs.path, 'not'], + }).safeParse(value).success, + 'Invalid input: Should NOT be valid against schema', + ); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-null.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-null.ts new file mode 100644 index 0000000000..86dbfea439 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-null.ts @@ -0,0 +1,7 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; + +export const parseNull = (_jsonSchema: JsonSchemaObject & { type: 'null' }) => { + return z.null(); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-nullable.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-nullable.ts new file mode 100644 index 0000000000..cfc575e9c7 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-nullable.ts @@ -0,0 +1,10 @@ +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, Refs } from '../types'; +import { omit } from '../utils/omit'; + +/** + * For compatibility with open api 3.0 nullable + */ +export const parseNullable = (jsonSchema: JsonSchemaObject & { nullable: true }, refs: Refs) => { + return parseSchema(omit(jsonSchema, 'nullable'), refs, true).nullable(); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-number.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-number.ts new file mode 100644 index 0000000000..504a453faf --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-number.ts @@ -0,0 +1,88 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; +import { extendSchemaWithMessage } from '../utils/extend-schema'; + +export const parseNumber = (jsonSchema: JsonSchemaObject & { type: 'number' | 'integer' }) => { + let zodSchema = z.number(); + + let isInteger = false; + if (jsonSchema.type === 'integer') { + isInteger = true; + zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'type', (zs, _, errorMsg) => + zs.int(errorMsg), + ); + } else if (jsonSchema.format === 'int64') { + isInteger = true; + zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'format', (zs, _, errorMsg) => + zs.int(errorMsg), + ); + } + + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'multipleOf', + (zs, multipleOf, errorMsg) => { + if (multipleOf === 1) { + if (isInteger) return zs; + + return zs.int(errorMsg); + } + + return zs.multipleOf(multipleOf, errorMsg); + }, + ); + + if (typeof jsonSchema.minimum === 'number') { + if (jsonSchema.exclusiveMinimum === true) { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'minimum', + (zs, minimum, errorMsg) => zs.gt(minimum, errorMsg), + ); + } else { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'minimum', + (zs, minimum, errorMsg) => zs.gte(minimum, errorMsg), + ); + } + } else if (typeof jsonSchema.exclusiveMinimum === 'number') { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'exclusiveMinimum', + (zs, exclusiveMinimum, errorMsg) => zs.gt(exclusiveMinimum as number, errorMsg), + ); + } + + if (typeof jsonSchema.maximum === 'number') { + if (jsonSchema.exclusiveMaximum === true) { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'maximum', + (zs, maximum, errorMsg) => zs.lt(maximum, errorMsg), + ); + } else { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'maximum', + (zs, maximum, errorMsg) => zs.lte(maximum, errorMsg), + ); + } + } else if (typeof jsonSchema.exclusiveMaximum === 'number') { + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'exclusiveMaximum', + (zs, exclusiveMaximum, errorMsg) => zs.lt(exclusiveMaximum as number, errorMsg), + ); + } + + return zodSchema; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-object.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-object.ts new file mode 100644 index 0000000000..6a87b7162b --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-object.ts @@ -0,0 +1,219 @@ +import * as z from 'zod'; + +import { parseAllOf } from './parse-all-of'; +import { parseAnyOf } from './parse-any-of'; +import { parseOneOf } from './parse-one-of'; +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, Refs } from '../types'; +import { its } from '../utils/its'; + +function parseObjectProperties(objectSchema: JsonSchemaObject & { type: 'object' }, refs: Refs) { + if (!objectSchema.properties) { + return undefined; + } + + const propertyKeys = Object.keys(objectSchema.properties); + if (propertyKeys.length === 0) { + return z.object({}); + } + + const properties: Record = {}; + + for (const key of propertyKeys) { + const propJsonSchema = objectSchema.properties[key]; + + const propZodSchema = parseSchema(propJsonSchema, { + ...refs, + path: [...refs.path, 'properties', key], + }); + + const hasDefault = typeof propJsonSchema === 'object' && propJsonSchema.default !== undefined; + + const required = Array.isArray(objectSchema.required) + ? objectSchema.required.includes(key) + : typeof propJsonSchema === 'object' && propJsonSchema.required === true; + + const isOptional = !hasDefault && !required; + + properties[key] = isOptional ? propZodSchema.optional() : propZodSchema; + } + + return z.object(properties); +} + +export function parseObject( + objectSchema: JsonSchemaObject & { type: 'object' }, + refs: Refs, +): z.ZodTypeAny { + const hasPatternProperties = Object.keys(objectSchema.patternProperties ?? {}).length > 0; + + const propertiesSchema: + | z.ZodObject, 'strip', z.ZodTypeAny> + | undefined = parseObjectProperties(objectSchema, refs); + let zodSchema: z.ZodTypeAny | undefined = propertiesSchema; + + const additionalProperties = + objectSchema.additionalProperties !== undefined + ? parseSchema(objectSchema.additionalProperties, { + ...refs, + path: [...refs.path, 'additionalProperties'], + }) + : undefined; + + if (objectSchema.patternProperties) { + const parsedPatternProperties = Object.fromEntries( + Object.entries(objectSchema.patternProperties).map(([key, value]) => { + return [ + key, + parseSchema(value, { + ...refs, + path: [...refs.path, 'patternProperties', key], + }), + ]; + }), + ); + const patternPropertyValues = Object.values(parsedPatternProperties); + + if (propertiesSchema) { + if (additionalProperties) { + zodSchema = propertiesSchema.catchall( + z.union([...patternPropertyValues, additionalProperties] as [z.ZodTypeAny, z.ZodTypeAny]), + ); + } else if (Object.keys(parsedPatternProperties).length > 1) { + zodSchema = propertiesSchema.catchall( + z.union(patternPropertyValues as [z.ZodTypeAny, z.ZodTypeAny]), + ); + } else { + zodSchema = propertiesSchema.catchall(patternPropertyValues[0]); + } + } else { + if (additionalProperties) { + zodSchema = z.record( + z.union([...patternPropertyValues, additionalProperties] as [z.ZodTypeAny, z.ZodTypeAny]), + ); + } else if (patternPropertyValues.length > 1) { + zodSchema = z.record(z.union(patternPropertyValues as [z.ZodTypeAny, z.ZodTypeAny])); + } else { + zodSchema = z.record(patternPropertyValues[0]); + } + } + + const objectPropertyKeys = new Set(Object.keys(objectSchema.properties ?? {})); + zodSchema = zodSchema.superRefine((value: Record, ctx) => { + for (const key in value) { + let wasMatched = objectPropertyKeys.has(key); + + for (const patternPropertyKey in objectSchema.patternProperties) { + const regex = new RegExp(patternPropertyKey); + if (key.match(regex)) { + wasMatched = true; + const result = parsedPatternProperties[patternPropertyKey].safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + } + + if (!wasMatched && additionalProperties) { + const result = additionalProperties.safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: 'Invalid input: must match catchall schema', + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + } + + let output: z.ZodTypeAny; + if (propertiesSchema) { + if (hasPatternProperties) { + output = zodSchema!; + } else if (additionalProperties) { + if (additionalProperties instanceof z.ZodNever) { + output = propertiesSchema.strict(); + } else { + output = propertiesSchema.catchall(additionalProperties); + } + } else { + output = zodSchema!; + } + } else { + if (hasPatternProperties) { + output = zodSchema!; + } else if (additionalProperties) { + output = z.record(additionalProperties); + } else { + output = z.record(z.any()); + } + } + + if (its.an.anyOf(objectSchema)) { + output = output.and( + parseAnyOf( + { + ...objectSchema, + anyOf: objectSchema.anyOf.map((x) => + typeof x === 'object' && + !x.type && + (x.properties ?? x.additionalProperties ?? x.patternProperties) + ? { ...x, type: 'object' } + : x, + ), + }, + refs, + ), + ); + } + + if (its.a.oneOf(objectSchema)) { + output = output.and( + parseOneOf( + { + ...objectSchema, + oneOf: objectSchema.oneOf.map((x) => + typeof x === 'object' && + !x.type && + (x.properties ?? x.additionalProperties ?? x.patternProperties) + ? { ...x, type: 'object' } + : x, + ), + }, + refs, + ), + ); + } + + if (its.an.allOf(objectSchema)) { + output = output.and( + parseAllOf( + { + ...objectSchema, + allOf: objectSchema.allOf.map((x) => + typeof x === 'object' && + !x.type && + (x.properties ?? x.additionalProperties ?? x.patternProperties) + ? { ...x, type: 'object' } + : x, + ), + }, + refs, + ), + ); + } + + return output; +} diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-one-of.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-one-of.ts new file mode 100644 index 0000000000..10931a9675 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-one-of.ts @@ -0,0 +1,41 @@ +import { z } from 'zod'; + +import { parseSchema } from './parse-schema'; +import type { JsonSchemaObject, JsonSchema, Refs } from '../types'; + +export const parseOneOf = (jsonSchema: JsonSchemaObject & { oneOf: JsonSchema[] }, refs: Refs) => { + if (!jsonSchema.oneOf.length) { + return z.any(); + } + + if (jsonSchema.oneOf.length === 1) { + return parseSchema(jsonSchema.oneOf[0], { + ...refs, + path: [...refs.path, 'oneOf', 0], + }); + } + + return z.any().superRefine((x, ctx) => { + const schemas = jsonSchema.oneOf.map((schema, i) => + parseSchema(schema, { + ...refs, + path: [...refs.path, 'oneOf', i], + }), + ); + + const unionErrors = schemas.reduce( + (errors, schema) => + ((result) => (result.error ? [...errors, result.error] : errors))(schema.safeParse(x)), + [], + ); + + if (schemas.length - unionErrors.length !== 1) { + ctx.addIssue({ + path: ctx.path, + code: 'invalid_union', + unionErrors, + message: 'Invalid input: Should pass single schema', + }); + } + }); +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-schema.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-schema.ts new file mode 100644 index 0000000000..24818bf490 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-schema.ts @@ -0,0 +1,130 @@ +import * as z from 'zod'; + +import { parseAllOf } from './parse-all-of'; +import { parseAnyOf } from './parse-any-of'; +import { parseArray } from './parse-array'; +import { parseBoolean } from './parse-boolean'; +import { parseConst } from './parse-const'; +import { parseDefault } from './parse-default'; +import { parseEnum } from './parse-enum'; +import { parseIfThenElse } from './parse-if-then-else'; +import { parseMultipleType } from './parse-multiple-type'; +import { parseNot } from './parse-not'; +import { parseNull } from './parse-null'; +import { parseNullable } from './parse-nullable'; +import { parseNumber } from './parse-number'; +import { parseObject } from './parse-object'; +import { parseOneOf } from './parse-one-of'; +import { parseString } from './parse-string'; +import type { ParserSelector, Refs, JsonSchemaObject, JsonSchema } from '../types'; +import { its } from '../utils/its'; + +const addDescribes = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { + if (jsonSchema.description) { + zodSchema = zodSchema.describe(jsonSchema.description); + } + + return zodSchema; +}; + +const addDefaults = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { + if (jsonSchema.default !== undefined) { + zodSchema = zodSchema.default(jsonSchema.default); + } + + return zodSchema; +}; + +const addAnnotations = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { + if (jsonSchema.readOnly) { + zodSchema = zodSchema.readonly(); + } + + return zodSchema; +}; + +const selectParser: ParserSelector = (schema, refs) => { + if (its.a.nullable(schema)) { + return parseNullable(schema, refs); + } else if (its.an.object(schema)) { + return parseObject(schema, refs); + } else if (its.an.array(schema)) { + return parseArray(schema, refs); + } else if (its.an.anyOf(schema)) { + return parseAnyOf(schema, refs); + } else if (its.an.allOf(schema)) { + return parseAllOf(schema, refs); + } else if (its.a.oneOf(schema)) { + return parseOneOf(schema, refs); + } else if (its.a.not(schema)) { + return parseNot(schema, refs); + } else if (its.an.enum(schema)) { + return parseEnum(schema); //<-- needs to come before primitives + } else if (its.a.const(schema)) { + return parseConst(schema); + } else if (its.a.multipleType(schema)) { + return parseMultipleType(schema, refs); + } else if (its.a.primitive(schema, 'string')) { + return parseString(schema); + } else if (its.a.primitive(schema, 'number') || its.a.primitive(schema, 'integer')) { + return parseNumber(schema); + } else if (its.a.primitive(schema, 'boolean')) { + return parseBoolean(schema); + } else if (its.a.primitive(schema, 'null')) { + return parseNull(schema); + } else if (its.a.conditional(schema)) { + return parseIfThenElse(schema, refs); + } else { + return parseDefault(schema); + } +}; + +export const parseSchema = ( + jsonSchema: JsonSchema, + refs: Refs = { seen: new Map(), path: [] }, + blockMeta?: boolean, +): z.ZodTypeAny => { + if (typeof jsonSchema !== 'object') return jsonSchema ? z.any() : z.never(); + + if (refs.parserOverride) { + const custom = refs.parserOverride(jsonSchema, refs); + + if (custom instanceof z.ZodType) { + return custom; + } + } + + let seen = refs.seen.get(jsonSchema); + + if (seen) { + if (seen.r !== undefined) { + return seen.r; + } + + if (refs.depth === undefined || seen.n >= refs.depth) { + return z.any(); + } + + seen.n += 1; + } else { + seen = { r: undefined, n: 0 }; + refs.seen.set(jsonSchema, seen); + } + + let parsedZodSchema = selectParser(jsonSchema, refs); + if (!blockMeta) { + if (!refs.withoutDescribes) { + parsedZodSchema = addDescribes(jsonSchema, parsedZodSchema); + } + + if (!refs.withoutDefaults) { + parsedZodSchema = addDefaults(jsonSchema, parsedZodSchema); + } + + parsedZodSchema = addAnnotations(jsonSchema, parsedZodSchema); + } + + seen.r = parsedZodSchema; + + return parsedZodSchema; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/parsers/parse-string.ts b/packages/@n8n/json-schema-to-zod/src/parsers/parse-string.ts new file mode 100644 index 0000000000..ea2be63c30 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/parsers/parse-string.ts @@ -0,0 +1,58 @@ +import { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; +import { extendSchemaWithMessage } from '../utils/extend-schema'; + +export const parseString = (jsonSchema: JsonSchemaObject & { type: 'string' }) => { + let zodSchema = z.string(); + + zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'format', (zs, format, errorMsg) => { + switch (format) { + case 'email': + return zs.email(errorMsg); + case 'ip': + return zs.ip(errorMsg); + case 'ipv4': + return zs.ip({ version: 'v4', message: errorMsg }); + case 'ipv6': + return zs.ip({ version: 'v6', message: errorMsg }); + case 'uri': + return zs.url(errorMsg); + case 'uuid': + return zs.uuid(errorMsg); + case 'date-time': + return zs.datetime({ offset: true, message: errorMsg }); + case 'time': + return zs.time(errorMsg); + case 'date': + return zs.date(errorMsg); + case 'binary': + return zs.base64(errorMsg); + case 'duration': + return zs.duration(errorMsg); + default: + return zs; + } + }); + + zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'contentEncoding', (zs, _, errorMsg) => + zs.base64(errorMsg), + ); + zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'pattern', (zs, pattern, errorMsg) => + zs.regex(new RegExp(pattern), errorMsg), + ); + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'minLength', + (zs, minLength, errorMsg) => zs.min(minLength, errorMsg), + ); + zodSchema = extendSchemaWithMessage( + zodSchema, + jsonSchema, + 'maxLength', + (zs, maxLength, errorMsg) => zs.max(maxLength, errorMsg), + ); + + return zodSchema; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/types.ts b/packages/@n8n/json-schema-to-zod/src/types.ts new file mode 100644 index 0000000000..bb342af230 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/types.ts @@ -0,0 +1,82 @@ +import type { ZodTypeAny } from 'zod'; + +export type Serializable = + | { [key: string]: Serializable } + | Serializable[] + | string + | number + | boolean + | null; + +export type JsonSchema = JsonSchemaObject | boolean; +export type JsonSchemaObject = { + // left permissive by design + type?: string | string[]; + + // object + properties?: { [key: string]: JsonSchema }; + additionalProperties?: JsonSchema; + unevaluatedProperties?: JsonSchema; + patternProperties?: { [key: string]: JsonSchema }; + minProperties?: number; + maxProperties?: number; + required?: string[] | boolean; + propertyNames?: JsonSchema; + + // array + items?: JsonSchema | JsonSchema[]; + additionalItems?: JsonSchema; + minItems?: number; + maxItems?: number; + uniqueItems?: boolean; + + // string + minLength?: number; + maxLength?: number; + pattern?: string; + format?: string; + + // number + minimum?: number; + maximum?: number; + exclusiveMinimum?: number | boolean; + exclusiveMaximum?: number | boolean; + multipleOf?: number; + + // unions + anyOf?: JsonSchema[]; + allOf?: JsonSchema[]; + oneOf?: JsonSchema[]; + + if?: JsonSchema; + then?: JsonSchema; + else?: JsonSchema; + + // shared + const?: Serializable; + enum?: Serializable[]; + + errorMessage?: { [key: string]: string | undefined }; + + description?: string; + default?: Serializable; + readOnly?: boolean; + not?: JsonSchema; + contentEncoding?: string; + nullable?: boolean; +}; + +export type ParserSelector = (schema: JsonSchemaObject, refs: Refs) => ZodTypeAny; +export type ParserOverride = (schema: JsonSchemaObject, refs: Refs) => ZodTypeAny | undefined; + +export type JsonSchemaToZodOptions = { + withoutDefaults?: boolean; + withoutDescribes?: boolean; + parserOverride?: ParserOverride; + depth?: number; +}; + +export type Refs = JsonSchemaToZodOptions & { + path: Array; + seen: Map; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/utils/extend-schema.ts b/packages/@n8n/json-schema-to-zod/src/utils/extend-schema.ts new file mode 100644 index 0000000000..1fd0ed720b --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/utils/extend-schema.ts @@ -0,0 +1,23 @@ +import type { z } from 'zod'; + +import type { JsonSchemaObject } from '../types'; + +export function extendSchemaWithMessage< + TZod extends z.ZodTypeAny, + TJson extends JsonSchemaObject, + TKey extends keyof TJson, +>( + zodSchema: TZod, + jsonSchema: TJson, + key: TKey, + extend: (zodSchema: TZod, value: NonNullable, errorMessage?: string) => TZod, +) { + const value = jsonSchema[key]; + + if (value !== undefined) { + const errorMessage = jsonSchema.errorMessage?.[key as string]; + return extend(zodSchema, value as NonNullable, errorMessage); + } + + return zodSchema; +} diff --git a/packages/@n8n/json-schema-to-zod/src/utils/half.ts b/packages/@n8n/json-schema-to-zod/src/utils/half.ts new file mode 100644 index 0000000000..810776e6c2 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/utils/half.ts @@ -0,0 +1,3 @@ +export const half = (arr: T[]): [T[], T[]] => { + return [arr.slice(0, arr.length / 2), arr.slice(arr.length / 2)]; +}; diff --git a/packages/@n8n/json-schema-to-zod/src/utils/its.ts b/packages/@n8n/json-schema-to-zod/src/utils/its.ts new file mode 100644 index 0000000000..494c1f6372 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/utils/its.ts @@ -0,0 +1,57 @@ +import type { JsonSchema, JsonSchemaObject, Serializable } from '../types'; + +export const its = { + an: { + object: (x: JsonSchemaObject): x is JsonSchemaObject & { type: 'object' } => + x.type === 'object', + array: (x: JsonSchemaObject): x is JsonSchemaObject & { type: 'array' } => x.type === 'array', + anyOf: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + anyOf: JsonSchema[]; + } => x.anyOf !== undefined, + allOf: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + allOf: JsonSchema[]; + } => x.allOf !== undefined, + enum: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + enum: Serializable | Serializable[]; + } => x.enum !== undefined, + }, + a: { + nullable: (x: JsonSchemaObject): x is JsonSchemaObject & { nullable: true } => + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (x as any).nullable === true, + multipleType: (x: JsonSchemaObject): x is JsonSchemaObject & { type: string[] } => + Array.isArray(x.type), + not: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + not: JsonSchema; + } => x.not !== undefined, + const: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + const: Serializable; + } => x.const !== undefined, + primitive: ( + x: JsonSchemaObject, + p: T, + ): x is JsonSchemaObject & { type: T } => x.type === p, + conditional: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + if: JsonSchema; + then: JsonSchema; + else: JsonSchema; + } => Boolean('if' in x && x.if && 'then' in x && 'else' in x && x.then && x.else), + oneOf: ( + x: JsonSchemaObject, + ): x is JsonSchemaObject & { + oneOf: JsonSchema[]; + } => x.oneOf !== undefined, + }, +}; diff --git a/packages/@n8n/json-schema-to-zod/src/utils/omit.ts b/packages/@n8n/json-schema-to-zod/src/utils/omit.ts new file mode 100644 index 0000000000..af9d579fb6 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/src/utils/omit.ts @@ -0,0 +1,8 @@ +export const omit = (obj: T, ...keys: K[]): Omit => + Object.keys(obj).reduce((acc: Record, key) => { + if (!keys.includes(key as K)) { + acc[key] = obj[key as K]; + } + + return acc; + }, {}) as Omit; diff --git a/packages/@n8n/json-schema-to-zod/test/all.json b/packages/@n8n/json-schema-to-zod/test/all.json new file mode 100644 index 0000000000..f270ca3fa1 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/all.json @@ -0,0 +1,143 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "properties": { + "allOf": { + "allOf": [ + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "anyOf": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "oneOf": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "array": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "maxItems": 3 + }, + "tuple": { + "type": "array", + "items": [ + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + } + ], + "minItems": 2, + "maxItems": 3 + }, + "const": { + "const": "xbox" + }, + "enum": { + "enum": ["ps4", "ps5"] + }, + "ifThenElse": { + "if": { + "type": "string" + }, + "then": { + "const": "x" + }, + "else": { + "enum": [1, 2, 3] + } + }, + "null": { + "type": "null" + }, + "multiple": { + "type": ["array", "boolean"] + }, + "objAdditionalTrue": { + "type": "object", + "properties": { + "x": { + "type": "string" + } + }, + "additionalProperties": true + }, + "objAdditionalFalse": { + "type": "object", + "properties": { + "x": { + "type": "string" + } + }, + "additionalProperties": false + }, + "objAdditionalNumber": { + "type": "object", + "properties": { + "x": { + "type": "string" + } + }, + "additionalProperties": { + "type": "number" + } + }, + "objAdditionalOnly": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "patternProps": { + "type": "object", + "patternProperties": { + "^x": { + "type": "string" + }, + "^y": { + "type": "number" + } + }, + "properties": { + "z": { + "type": "string" + } + }, + "additionalProperties": false + } + } +} diff --git a/packages/@n8n/json-schema-to-zod/test/extend-expect.ts b/packages/@n8n/json-schema-to-zod/test/extend-expect.ts new file mode 100644 index 0000000000..5196e43416 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/extend-expect.ts @@ -0,0 +1,16 @@ +import type { z } from 'zod'; + +expect.extend({ + toMatchZod(this: jest.MatcherContext, actual: z.ZodTypeAny, expected: z.ZodTypeAny) { + const actualSerialized = JSON.stringify(actual._def, null, 2); + const expectedSerialized = JSON.stringify(expected._def, null, 2); + const pass = this.equals(actualSerialized, expectedSerialized); + + return { + pass, + message: pass + ? () => `Expected ${actualSerialized} not to match ${expectedSerialized}` + : () => `Expected ${actualSerialized} to match ${expectedSerialized}`, + }; + }, +}); diff --git a/packages/@n8n/json-schema-to-zod/test/jest.d.ts b/packages/@n8n/json-schema-to-zod/test/jest.d.ts new file mode 100644 index 0000000000..dff5a5fa4c --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/jest.d.ts @@ -0,0 +1,5 @@ +namespace jest { + interface Matchers { + toMatchZod(expected: unknown): T; + } +} diff --git a/packages/@n8n/json-schema-to-zod/test/json-schema-to-zod.test.ts b/packages/@n8n/json-schema-to-zod/test/json-schema-to-zod.test.ts new file mode 100644 index 0000000000..5f383ae71b --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/json-schema-to-zod.test.ts @@ -0,0 +1,106 @@ +import type { JSONSchema4, JSONSchema6Definition, JSONSchema7Definition } from 'json-schema'; +import { z } from 'zod'; + +import { jsonSchemaToZod } from '../src'; + +describe('jsonSchemaToZod', () => { + test('should accept json schema 7 and 4', () => { + const schema = { type: 'string' } as unknown; + + expect(jsonSchemaToZod(schema as JSONSchema4)); + expect(jsonSchemaToZod(schema as JSONSchema6Definition)); + expect(jsonSchemaToZod(schema as JSONSchema7Definition)); + }); + + test('can exclude defaults', () => { + expect( + jsonSchemaToZod( + { + type: 'string', + default: 'foo', + }, + { withoutDefaults: true }, + ), + ).toMatchZod(z.string()); + }); + + test('should include describes', () => { + expect( + jsonSchemaToZod({ + type: 'string', + description: 'foo', + }), + ).toMatchZod(z.string().describe('foo')); + }); + + test('can exclude describes', () => { + expect( + jsonSchemaToZod( + { + type: 'string', + description: 'foo', + }, + { + withoutDescribes: true, + }, + ), + ).toMatchZod(z.string()); + }); + + test('will remove optionality if default is present', () => { + expect( + jsonSchemaToZod({ + type: 'object', + properties: { + prop: { + type: 'string', + default: 'def', + }, + }, + }), + ).toMatchZod(z.object({ prop: z.string().default('def') })); + }); + + test('will handle falsy defaults', () => { + expect( + jsonSchemaToZod({ + type: 'boolean', + default: false, + }), + ).toMatchZod(z.boolean().default(false)); + }); + + test('will ignore undefined as default', () => { + expect( + jsonSchemaToZod({ + type: 'null', + default: undefined, + }), + ).toMatchZod(z.null()); + }); + + test('should be possible to define a custom parser', () => { + expect( + jsonSchemaToZod( + { + allOf: [{ type: 'string' }, { type: 'number' }, { type: 'boolean', description: 'foo' }], + }, + { + parserOverride: (schema, refs) => { + if ( + refs.path.length === 2 && + refs.path[0] === 'allOf' && + refs.path[1] === 2 && + schema.type === 'boolean' && + schema.description === 'foo' + ) { + return z.null(); + } + + return undefined; + }, + }, + ), + ).toMatchZod(z.intersection(z.string(), z.intersection(z.number(), z.null()))); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-all-of.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-all-of.test.ts new file mode 100644 index 0000000000..546572255c --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-all-of.test.ts @@ -0,0 +1,48 @@ +import { z } from 'zod'; + +import { parseAllOf } from '../../src/parsers/parse-all-of'; + +describe('parseAllOf', () => { + test('should create never if empty', () => { + expect( + parseAllOf( + { + allOf: [], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.never()); + }); + + test('should handle true values', () => { + expect( + parseAllOf( + { + allOf: [{ type: 'string' }, true], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.intersection(z.string(), z.any())); + }); + + test('should handle false values', () => { + expect( + parseAllOf( + { + allOf: [{ type: 'string' }, false], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.intersection( + z.string(), + z + .any() + .refine( + (value) => !z.any().safeParse(value).success, + 'Invalid input: Should NOT be valid against schema', + ), + ), + ); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-any-of.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-any-of.test.ts new file mode 100644 index 0000000000..72abcab047 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-any-of.test.ts @@ -0,0 +1,31 @@ +import { z } from 'zod'; + +import { parseAnyOf } from '../../src/parsers/parse-any-of'; + +describe('parseAnyOf', () => { + test('should create a union from two or more schemas', () => { + expect( + parseAnyOf( + { + anyOf: [ + { + type: 'string', + }, + { type: 'number' }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.union([z.string(), z.number()])); + }); + + test('should extract a single schema', () => { + expect(parseAnyOf({ anyOf: [{ type: 'string' }] }, { path: [], seen: new Map() })).toMatchZod( + z.string(), + ); + }); + + test('should return z.any() if array is empty', () => { + expect(parseAnyOf({ anyOf: [] }, { path: [], seen: new Map() })).toMatchZod(z.any()); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-array.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-array.test.ts new file mode 100644 index 0000000000..b96df3958c --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-array.test.ts @@ -0,0 +1,68 @@ +import { z } from 'zod'; + +import { parseArray } from '../../src/parsers/parse-array'; + +describe('parseArray', () => { + test('should create tuple with items array', () => { + expect( + parseArray( + { + type: 'array', + items: [ + { + type: 'string', + }, + { + type: 'number', + }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.tuple([z.string(), z.number()])); + }); + + test('should create array with items object', () => { + expect( + parseArray( + { + type: 'array', + items: { + type: 'string', + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.array(z.string())); + }); + + test('should create min for minItems', () => { + expect( + parseArray( + { + type: 'array', + minItems: 2, + items: { + type: 'string', + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.array(z.string()).min(2)); + }); + + test('should create max for maxItems', () => { + expect( + parseArray( + { + type: 'array', + maxItems: 2, + items: { + type: 'string', + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.array(z.string()).max(2)); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-const.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-const.test.ts new file mode 100644 index 0000000000..b4f7a5afd5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-const.test.ts @@ -0,0 +1,13 @@ +import { z } from 'zod'; + +import { parseConst } from '../../src/parsers/parse-const'; + +describe('parseConst', () => { + test('should handle falsy constants', () => { + expect( + parseConst({ + const: false, + }), + ).toMatchZod(z.literal(false)); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-enum.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-enum.test.ts new file mode 100644 index 0000000000..2ed00e3def --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-enum.test.ts @@ -0,0 +1,36 @@ +import { z } from 'zod'; + +import { parseEnum } from '../../src/parsers/parse-enum'; + +describe('parseEnum', () => { + test('should create never with empty enum', () => { + expect( + parseEnum({ + enum: [], + }), + ).toMatchZod(z.never()); + }); + + test('should create literal with single item enum', () => { + expect( + parseEnum({ + enum: ['someValue'], + }), + ).toMatchZod(z.literal('someValue')); + }); + + test('should create enum array with string enums', () => { + expect( + parseEnum({ + enum: ['someValue', 'anotherValue'], + }), + ).toMatchZod(z.enum(['someValue', 'anotherValue'])); + }); + test('should create union with mixed enums', () => { + expect( + parseEnum({ + enum: ['someValue', 57], + }), + ).toMatchZod(z.union([z.literal('someValue'), z.literal(57)])); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-not.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-not.test.ts new file mode 100644 index 0000000000..f1bc1d7ab2 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-not.test.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +import { parseNot } from '../../src/parsers/parse-not'; + +describe('parseNot', () => { + test('parseNot', () => { + expect( + parseNot( + { + not: { + type: 'string', + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z + .any() + .refine( + (value) => !z.string().safeParse(value).success, + 'Invalid input: Should NOT be valid against schema', + ), + ); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-nullable.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-nullable.test.ts new file mode 100644 index 0000000000..046c3f41a1 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-nullable.test.ts @@ -0,0 +1,18 @@ +import { z } from 'zod'; + +import { parseSchema } from '../../src/parsers/parse-schema'; + +describe('parseNullable', () => { + test('parseSchema should not add default twice', () => { + expect( + parseSchema( + { + type: 'string', + nullable: true, + default: null, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.string().nullable().default(null)); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-number.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-number.test.ts new file mode 100644 index 0000000000..7b3cdf4ded --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-number.test.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; + +import { parseNumber } from '../../src/parsers/parse-number'; + +describe('parseNumber', () => { + test('should handle integer', () => { + expect( + parseNumber({ + type: 'integer', + }), + ).toMatchZod(z.number().int()); + + expect( + parseNumber({ + type: 'integer', + multipleOf: 1, + }), + ).toMatchZod(z.number().int()); + + expect( + parseNumber({ + type: 'number', + multipleOf: 1, + }), + ).toMatchZod(z.number().int()); + }); + + test('should handle maximum with exclusiveMinimum', () => { + expect( + parseNumber({ + type: 'number', + exclusiveMinimum: true, + minimum: 2, + }), + ).toMatchZod(z.number().gt(2)); + }); + + test('should handle maximum with exclusiveMinimum', () => { + expect( + parseNumber({ + type: 'number', + minimum: 2, + }), + ).toMatchZod(z.number().gte(2)); + }); + + test('should handle maximum with exclusiveMaximum', () => { + expect( + parseNumber({ + type: 'number', + exclusiveMaximum: true, + maximum: 2, + }), + ).toMatchZod(z.number().lt(2)); + }); + + test('should handle numeric exclusiveMaximum', () => { + expect( + parseNumber({ + type: 'number', + exclusiveMaximum: 2, + }), + ).toMatchZod(z.number().lt(2)); + }); + + test('should accept errorMessage', () => { + expect( + parseNumber({ + type: 'number', + format: 'int64', + exclusiveMinimum: 0, + maximum: 2, + multipleOf: 2, + errorMessage: { + format: 'ayy', + multipleOf: 'lmao', + exclusiveMinimum: 'deez', + maximum: 'nuts', + }, + }), + ).toMatchZod(z.number().int('ayy').multipleOf(2, 'lmao').gt(0, 'deez').lte(2, 'nuts')); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-object.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-object.test.ts new file mode 100644 index 0000000000..00a2e194cd --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-object.test.ts @@ -0,0 +1,904 @@ +/* eslint-disable n8n-local-rules/no-skipped-tests */ +import type { JSONSchema7 } from 'json-schema'; +import { z, ZodError } from 'zod'; + +import { parseObject } from '../../src/parsers/parse-object'; + +describe('parseObject', () => { + test('should handle with missing properties', () => { + expect( + parseObject( + { + type: 'object', + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.record(z.any())); + }); + + test('should handle with empty properties', () => { + expect( + parseObject( + { + type: 'object', + properties: {}, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({})); + }); + + test('With properties - should handle optional and required properties', () => { + expect( + parseObject( + { + type: 'object', + required: ['myRequiredString'], + properties: { + myOptionalString: { + type: 'string', + }, + myRequiredString: { + type: 'string', + }, + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.object({ myOptionalString: z.string().optional(), myRequiredString: z.string() }), + ); + }); + + test('With properties - should handle additionalProperties when set to false', () => { + expect( + parseObject( + { + type: 'object', + required: ['myString'], + properties: { + myString: { + type: 'string', + }, + }, + additionalProperties: false, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({ myString: z.string() }).strict()); + }); + + test('With properties - should handle additionalProperties when set to true', () => { + expect( + parseObject( + { + type: 'object', + required: ['myString'], + properties: { + myString: { + type: 'string', + }, + }, + additionalProperties: true, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({ myString: z.string() }).catchall(z.any())); + }); + + test('With properties - should handle additionalProperties when provided a schema', () => { + expect( + parseObject( + { + type: 'object', + required: ['myString'], + properties: { + myString: { + type: 'string', + }, + }, + additionalProperties: { type: 'number' }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({ myString: z.string() }).catchall(z.number())); + }); + + test('Without properties - should handle additionalProperties when set to false', () => { + expect( + parseObject( + { + type: 'object', + additionalProperties: false, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.record(z.never())); + }); + + test('Without properties - should handle additionalProperties when set to true', () => { + expect( + parseObject( + { + type: 'object', + additionalProperties: true, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.record(z.any())); + }); + + test('Without properties - should handle additionalProperties when provided a schema', () => { + expect( + parseObject( + { + type: 'object', + additionalProperties: { type: 'number' }, + }, + + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.record(z.number())); + }); + + test('Without properties - should include falsy defaults', () => { + expect( + parseObject( + { + type: 'object', + properties: { + s: { + type: 'string', + default: '', + }, + }, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({ s: z.string().default('') })); + }); + + test('eh', () => { + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + anyOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + { + required: ['c'], + properties: { + c: { + type: 'string', + }, + }, + }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z + .object({ a: z.string() }) + .and(z.union([z.object({ b: z.string() }), z.object({ c: z.string() })])), + ); + + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + anyOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + {}, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.object({ a: z.string() }).and(z.union([z.object({ b: z.string() }), z.any()]))); + + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + oneOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + { + required: ['c'], + properties: { + c: { + type: 'string', + }, + }, + }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.object({ a: z.string() }).and( + z.any().superRefine((x, ctx) => { + const schemas = [z.object({ b: z.string() }), z.object({ c: z.string() })]; + const errors = schemas.reduce( + (errors, schema) => + ((result) => (result.error ? [...errors, result.error] : errors))( + schema.safeParse(x), + ), + [], + ); + if (schemas.length - errors.length !== 1) { + ctx.addIssue({ + path: ctx.path, + code: 'invalid_union', + unionErrors: errors, + message: 'Invalid input: Should pass single schema', + }); + } + }), + ), + ); + + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + oneOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + {}, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.object({ a: z.string() }).and( + z.any().superRefine((x, ctx) => { + const schemas = [z.object({ b: z.string() }), z.any()]; + const errors = schemas.reduce( + (errors, schema) => + ((result) => (result.error ? [...errors, result.error] : errors))( + schema.safeParse(x), + ), + [], + ); + if (schemas.length - errors.length !== 1) { + ctx.addIssue({ + path: ctx.path, + code: 'invalid_union', + unionErrors: errors, + message: 'Invalid input: Should pass single schema', + }); + } + }), + ), + ); + + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + allOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + { + required: ['c'], + properties: { + c: { + type: 'string', + }, + }, + }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z + .object({ a: z.string() }) + .and(z.intersection(z.object({ b: z.string() }), z.object({ c: z.string() }))), + ); + + expect( + parseObject( + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + allOf: [ + { + required: ['b'], + properties: { + b: { + type: 'string', + }, + }, + }, + {}, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.object({ a: z.string() }).and(z.intersection(z.object({ b: z.string() }), z.any())), + ); + }); + + const run = (zodSchema: z.ZodTypeAny, data: unknown) => zodSchema.safeParse(data); + + test('Functional tests - run', () => { + expect(run(z.string(), 'hello')).toEqual({ + success: true, + data: 'hello', + }); + }); + + test('Functional tests - properties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + b: { + type: 'number', + }, + }, + }; + + const expected = z.object({ a: z.string(), b: z.number().optional() }); + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + + expect(run(result, { a: 'hello' })).toEqual({ + success: true, + data: { + a: 'hello', + }, + }); + + expect(run(result, { a: 'hello', b: 123 })).toEqual({ + success: true, + data: { + a: 'hello', + b: 123, + }, + }); + + expect(run(result, { b: 'hello', x: true })).toEqual({ + success: false, + error: new ZodError([ + { + code: 'invalid_type', + expected: 'string', + received: 'undefined', + path: ['a'], + message: 'Required', + }, + { + code: 'invalid_type', + expected: 'number', + received: 'string', + path: ['b'], + message: 'Expected number, received string', + }, + ]), + }); + }); + + test('Functional tests - properties and additionalProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + b: { + type: 'number', + }, + }, + additionalProperties: { type: 'boolean' }, + }; + + const expected = z.object({ a: z.string(), b: z.number().optional() }).catchall(z.boolean()); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + + expect(run(result, { b: 'hello', x: 'true' })).toEqual({ + success: false, + error: new ZodError([ + { + code: 'invalid_type', + expected: 'string', + received: 'undefined', + path: ['a'], + message: 'Required', + }, + { + code: 'invalid_type', + expected: 'number', + received: 'string', + path: ['b'], + message: 'Expected number, received string', + }, + { + code: 'invalid_type', + expected: 'boolean', + received: 'string', + path: ['x'], + message: 'Expected boolean, received string', + }, + ]), + }); + }); + + test('Functional tests - properties and single-item patternProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + b: { + type: 'number', + }, + }, + patternProperties: { + '\\.': { type: 'array' }, + }, + }; + + const expected = z + .object({ a: z.string(), b: z.number().optional() }) + .catchall(z.array(z.any())) + .superRefine((value, ctx) => { + for (const key in value) { + if (key.match(new RegExp('\\\\.'))) { + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + + expect(run(result, { a: 'a', b: 2, '.': [] })).toEqual({ + success: true, + data: { a: 'a', b: 2, '.': [] }, + }); + + expect(run(result, { a: 'a', b: 2, '.': '[]' })).toEqual({ + success: false, + error: new ZodError([ + { + code: 'invalid_type', + expected: 'array', + received: 'string', + path: ['.'], + message: 'Expected array, received string', + }, + ]), + }); + }); + + test('Functional tests - properties, additionalProperties and patternProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + b: { + type: 'number', + }, + }, + additionalProperties: { type: 'boolean' }, + patternProperties: { + '\\.': { type: 'array' }, + '\\,': { type: 'array', minItems: 1 }, + }, + }; + + const expected = z + .object({ a: z.string(), b: z.number().optional() }) + .catchall(z.union([z.array(z.any()), z.array(z.any()).min(1), z.boolean()])) + .superRefine((value, ctx) => { + for (const key in value) { + let evaluated = ['a', 'b'].includes(key); + if (key.match(new RegExp('\\\\.'))) { + evaluated = true; + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (key.match(new RegExp('\\\\,'))) { + evaluated = true; + const result = z.array(z.any()).min(1).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (!evaluated) { + const result = z.boolean().safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: 'Invalid input: must match catchall schema', + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + }); + + test('Functional tests - additionalProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + additionalProperties: { type: 'boolean' }, + }; + + const expected = z.record(z.boolean()); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + }); + + test('Functional tests - additionalProperties and patternProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + additionalProperties: { type: 'boolean' }, + patternProperties: { + '\\.': { type: 'array' }, + '\\,': { type: 'array', minItems: 1 }, + }, + }; + + const expected = z + .record(z.union([z.array(z.any()), z.array(z.any()).min(1), z.boolean()])) + .superRefine((value, ctx) => { + for (const key in value) { + let evaluated = false; + if (key.match(new RegExp('\\\\.'))) { + evaluated = true; + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (key.match(new RegExp('\\\\,'))) { + evaluated = true; + const result = z.array(z.any()).min(1).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (!evaluated) { + const result = z.boolean().safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: 'Invalid input: must match catchall schema', + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + + expect(run(result, { x: true, '.': [], ',': [] })).toEqual({ + success: false, + error: new ZodError([ + { + path: [','], + code: 'custom', + message: 'Invalid input: Key matching regex /,/ must match schema', + params: { + issues: [ + { + code: 'too_small', + minimum: 1, + type: 'array', + inclusive: true, + exact: false, + message: 'Array must contain at least 1 element(s)', + path: [], + }, + ], + }, + }, + ]), + }); + }); + + test('Functional tests - single-item patternProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + patternProperties: { + '\\.': { type: 'array' }, + }, + }; + + const expected = z.record(z.array(z.any())).superRefine((value, ctx) => { + for (const key in value) { + if (key.match(new RegExp('\\\\.'))) { + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + }); + + test('Functional tests - patternProperties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + patternProperties: { + '\\.': { type: 'array' }, + '\\,': { type: 'array', minItems: 1 }, + }, + }; + + const expected = z + .record(z.union([z.array(z.any()), z.array(z.any()).min(1)])) + .superRefine((value, ctx) => { + for (const key in value) { + if (key.match(new RegExp('\\.'))) { + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (key.match(new RegExp('\\,'))) { + const result = z.array(z.any()).min(1).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(run(result, { '.': [] })).toEqual({ + success: true, + data: { '.': [] }, + }); + + expect(run(result, { ',': [] })).toEqual({ + success: false, + error: new ZodError([ + { + path: [','], + code: 'custom', + message: 'Invalid input: Key matching regex /,/ must match schema', + params: { + issues: [ + { + code: 'too_small', + minimum: 1, + type: 'array', + inclusive: true, + exact: false, + message: 'Array must contain at least 1 element(s)', + path: [], + }, + ], + }, + }, + ]), + }); + + expect(result).toMatchZod(expected); + }); + + test('Functional tests - patternProperties and properties', () => { + const schema: JSONSchema7 & { type: 'object' } = { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + b: { + type: 'number', + }, + }, + patternProperties: { + '\\.': { type: 'array' }, + '\\,': { type: 'array', minItems: 1 }, + }, + }; + + const expected = z + .object({ a: z.string(), b: z.number().optional() }) + .catchall(z.union([z.array(z.any()), z.array(z.any()).min(1)])) + .superRefine((value, ctx) => { + for (const key in value) { + if (key.match(new RegExp('\\.'))) { + const result = z.array(z.any()).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + if (key.match(new RegExp('\\,'))) { + const result = z.array(z.any()).min(1).safeParse(value[key]); + if (!result.success) { + ctx.addIssue({ + path: [...ctx.path, key], + code: 'custom', + message: `Invalid input: Key matching regex /${key}/ must match schema`, + params: { + issues: result.error.issues, + }, + }); + } + } + } + }); + + const result = parseObject(schema, { path: [], seen: new Map() }); + + expect(result).toMatchZod(expected); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-one-of.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-one-of.test.ts new file mode 100644 index 0000000000..3295200576 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-one-of.test.ts @@ -0,0 +1,48 @@ +import { z } from 'zod'; + +import { parseOneOf } from '../../src/parsers/parse-one-of'; + +describe('parseOneOf', () => { + test('should create a union from two or more schemas', () => { + expect( + parseOneOf( + { + oneOf: [ + { + type: 'string', + }, + { type: 'number' }, + ], + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod( + z.any().superRefine((x, ctx) => { + const schemas = [z.string(), z.number()]; + const errors = schemas.reduce( + (errors, schema) => + ((result) => (result.error ? [...errors, result.error] : errors))(schema.safeParse(x)), + [], + ); + if (schemas.length - errors.length !== 1) { + ctx.addIssue({ + path: ctx.path, + code: 'invalid_union', + unionErrors: errors, + message: 'Invalid input: Should pass single schema', + }); + } + }), + ); + }); + + test('should extract a single schema', () => { + expect(parseOneOf({ oneOf: [{ type: 'string' }] }, { path: [], seen: new Map() })).toMatchZod( + z.string(), + ); + }); + + test('should return z.any() if array is empty', () => { + expect(parseOneOf({ oneOf: [] }, { path: [], seen: new Map() })).toMatchZod(z.any()); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-schema.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-schema.test.ts new file mode 100644 index 0000000000..c0f0899e28 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-schema.test.ts @@ -0,0 +1,113 @@ +import { z } from 'zod'; + +import { parseSchema } from '../../src/parsers/parse-schema'; + +describe('parseSchema', () => { + test('should be usable without providing refs', () => { + expect(parseSchema({ type: 'string' })).toMatchZod(z.string()); + }); + + test('should return a seen and processed ref', () => { + const seen = new Map(); + const schema = { + type: 'object', + properties: { + prop: { + type: 'string', + }, + }, + }; + expect(parseSchema(schema, { seen, path: [] })); + expect(parseSchema(schema, { seen, path: [] })); + }); + + test('should be possible to describe a readonly schema', () => { + expect(parseSchema({ type: 'string', readOnly: true })).toMatchZod(z.string().readonly()); + }); + + test('should handle nullable', () => { + expect( + parseSchema( + { + type: 'string', + nullable: true, + }, + { path: [], seen: new Map() }, + ), + ).toMatchZod(z.string().nullable()); + }); + + test('should handle enum', () => { + expect(parseSchema({ enum: ['someValue', 57] })).toMatchZod( + z.union([z.literal('someValue'), z.literal(57)]), + ); + }); + + test('should handle multiple type', () => { + expect(parseSchema({ type: ['string', 'number'] })).toMatchZod( + z.union([z.string(), z.number()]), + ); + }); + + test('should handle if-then-else type', () => { + expect( + parseSchema({ + if: { type: 'string' }, + then: { type: 'number' }, + else: { type: 'boolean' }, + }), + ).toMatchZod( + z.union([z.number(), z.boolean()]).superRefine((value, ctx) => { + const result = z.string().safeParse(value).success + ? z.number().safeParse(value) + : z.boolean().safeParse(value); + if (!result.success) { + result.error.errors.forEach((error) => ctx.addIssue(error)); + } + }), + ); + }); + + test('should handle anyOf', () => { + expect( + parseSchema({ + anyOf: [ + { + type: 'string', + }, + { type: 'number' }, + ], + }), + ).toMatchZod(z.union([z.string(), z.number()])); + }); + + test('should handle oneOf', () => { + expect( + parseSchema({ + oneOf: [ + { + type: 'string', + }, + { type: 'number' }, + ], + }), + ).toMatchZod( + z.any().superRefine((x, ctx) => { + const schemas = [z.string(), z.number()]; + const errors = schemas.reduce( + (errors, schema) => + ((result) => (result.error ? [...errors, result.error] : errors))(schema.safeParse(x)), + [], + ); + if (schemas.length - errors.length !== 1) { + ctx.addIssue({ + path: ctx.path, + code: 'invalid_union', + unionErrors: errors, + message: 'Invalid input: Should pass single schema', + }); + } + }), + ); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/parsers/parse-string.test.ts b/packages/@n8n/json-schema-to-zod/test/parsers/parse-string.test.ts new file mode 100644 index 0000000000..5e53135c2d --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/parsers/parse-string.test.ts @@ -0,0 +1,152 @@ +import { z } from 'zod'; + +import { parseString } from '../../src/parsers/parse-string'; + +describe('parseString', () => { + const run = (schema: z.ZodString, data: unknown) => schema.safeParse(data); + + test('DateTime format', () => { + const datetime = '2018-11-13T20:20:39Z'; + + const code = parseString({ + type: 'string', + format: 'date-time', + errorMessage: { format: 'hello' }, + }); + + expect(code).toMatchZod(z.string().datetime({ offset: true, message: 'hello' })); + + expect(run(code, datetime)).toEqual({ success: true, data: datetime }); + }); + + test('email', () => { + expect( + parseString({ + type: 'string', + format: 'email', + }), + ).toMatchZod(z.string().email()); + }); + + test('ip', () => { + expect( + parseString({ + type: 'string', + format: 'ip', + }), + ).toMatchZod(z.string().ip()); + + expect( + parseString({ + type: 'string', + format: 'ipv6', + }), + ).toMatchZod(z.string().ip({ version: 'v6' })); + }); + + test('uri', () => { + expect( + parseString({ + type: 'string', + format: 'uri', + }), + ).toMatchZod(z.string().url()); + }); + + test('uuid', () => { + expect( + parseString({ + type: 'string', + format: 'uuid', + }), + ).toMatchZod(z.string().uuid()); + }); + + test('time', () => { + expect( + parseString({ + type: 'string', + format: 'time', + }), + ).toMatchZod(z.string().time()); + }); + + test('date', () => { + expect( + parseString({ + type: 'string', + format: 'date', + }), + ).toMatchZod(z.string().date()); + }); + + test('duration', () => { + expect( + parseString({ + type: 'string', + format: 'duration', + }), + ).toMatchZod(z.string().duration()); + }); + + test('base64', () => { + expect( + parseString({ + type: 'string', + contentEncoding: 'base64', + }), + ).toMatchZod(z.string().base64()); + + expect( + parseString({ + type: 'string', + contentEncoding: 'base64', + errorMessage: { + contentEncoding: 'x', + }, + }), + ).toMatchZod(z.string().base64('x')); + + expect( + parseString({ + type: 'string', + format: 'binary', + }), + ).toMatchZod(z.string().base64()); + + expect( + parseString({ + type: 'string', + format: 'binary', + errorMessage: { + format: 'x', + }, + }), + ).toMatchZod(z.string().base64('x')); + }); + + test('should accept errorMessage', () => { + expect( + parseString({ + type: 'string', + format: 'ipv4', + pattern: 'x', + minLength: 1, + maxLength: 2, + errorMessage: { + format: 'ayy', + pattern: 'lmao', + minLength: 'deez', + maxLength: 'nuts', + }, + }), + ).toMatchZod( + z + .string() + .ip({ version: 'v4', message: 'ayy' }) + .regex(new RegExp('x'), 'lmao') + .min(1, 'deez') + .max(2, 'nuts'), + ); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/utils/half.test.ts b/packages/@n8n/json-schema-to-zod/test/utils/half.test.ts new file mode 100644 index 0000000000..afd9ee85fd --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/utils/half.test.ts @@ -0,0 +1,15 @@ +import { half } from '../../src/utils/half'; + +describe('half', () => { + test('half', () => { + const [a, b] = half(['A', 'B', 'C', 'D', 'E']); + + if (1 < 0) { + // type should be string + a[0].endsWith(''); + } + + expect(a).toEqual(['A', 'B']); + expect(b).toEqual(['C', 'D', 'E']); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/test/utils/omit.test.ts b/packages/@n8n/json-schema-to-zod/test/utils/omit.test.ts new file mode 100644 index 0000000000..f5f51313f5 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/test/utils/omit.test.ts @@ -0,0 +1,27 @@ +import { omit } from '../../src/utils/omit'; + +describe('omit', () => { + test('omit', () => { + const input = { + a: true, + b: true, + }; + + omit( + input, + 'b', + // @ts-expect-error + 'c', + ); + + const output = omit(input, 'b'); + + // @ts-expect-error + output.b; + + expect(output.a).toBe(true); + + // @ts-expect-error + expect(output.b).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/json-schema-to-zod/tsconfig.cjs.json b/packages/@n8n/json-schema-to-zod/tsconfig.cjs.json new file mode 100644 index 0000000000..2a17765d74 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/tsconfig.cjs.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "commonjs", + "outDir": "dist/cjs", + "strict": true, + "skipLibCheck": true, + "esModuleInterop": true + }, + "include": ["src"] +} diff --git a/packages/@n8n/json-schema-to-zod/tsconfig.esm.json b/packages/@n8n/json-schema-to-zod/tsconfig.esm.json new file mode 100644 index 0000000000..21f4508341 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/tsconfig.esm.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "es2020", + "moduleResolution": "node", + "outDir": "dist/esm", + "strict": true, + "skipLibCheck": true, + "esModuleInterop": true + }, + "include": ["src"] +} diff --git a/packages/@n8n/json-schema-to-zod/tsconfig.json b/packages/@n8n/json-schema-to-zod/tsconfig.json new file mode 100644 index 0000000000..f8e6508e74 --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": ["../../../tsconfig.json"], + "compilerOptions": { + "rootDir": ".", + "baseUrl": "src", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "esModuleInterop": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/@n8n/json-schema-to-zod/tsconfig.types.json b/packages/@n8n/json-schema-to-zod/tsconfig.types.json new file mode 100644 index 0000000000..63451df65a --- /dev/null +++ b/packages/@n8n/json-schema-to-zod/tsconfig.types.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist/types", + "strict": true, + "skipLibCheck": true, + "esModuleInterop": true + }, + "include": ["src"] +} diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts index a14e4195c9..0b27456db6 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts @@ -8,7 +8,7 @@ import type { INodeTypeDescription, INodeProperties, } from 'n8n-workflow'; -import { promptTypeOptions, textInput } from '../../../utils/descriptions'; + import { conversationalAgentProperties } from './agents/ConversationalAgent/description'; import { conversationalAgentExecute } from './agents/ConversationalAgent/execute'; import { openAiFunctionsAgentProperties } from './agents/OpenAiFunctionsAgent/description'; @@ -21,6 +21,7 @@ import { sqlAgentAgentProperties } from './agents/SqlAgent/description'; import { sqlAgentAgentExecute } from './agents/SqlAgent/execute'; import { toolsAgentProperties } from './agents/ToolsAgent/description'; import { toolsAgentExecute } from './agents/ToolsAgent/execute'; +import { promptTypeOptions, textInput } from '../../../utils/descriptions'; // Function used in the inputs expression to figure out which inputs to // display based on the agent type @@ -250,7 +251,7 @@ export class Agent implements INodeType { icon: 'fa:robot', iconColor: 'black', group: ['transform'], - version: [1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6], + version: [1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7], description: 'Generates an action plan and executes it. Can use external tools.', subtitle: "={{ { toolsAgent: 'Tools Agent', conversationalAgent: 'Conversational Agent', openAiFunctionsAgent: 'OpenAI Functions Agent', reActAgent: 'ReAct Agent', sqlAgent: 'SQL Agent', planAndExecuteAgent: 'Plan and Execute Agent' }[$parameter.agent] }}", @@ -351,6 +352,23 @@ export class Agent implements INodeType { }, }, }, + { + displayName: 'For more reliable structured output parsing, consider using the Tools agent', + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + hasOutputParser: [true], + agent: [ + 'conversationalAgent', + 'reActAgent', + 'planAndExecuteAgent', + 'openAiFunctionsAgent', + ], + }, + }, + }, { displayName: 'Require Specific Output Format', name: 'hasOutputParser', @@ -372,6 +390,7 @@ export class Agent implements INodeType { displayOptions: { show: { hasOutputParser: [true], + agent: ['toolsAgent'], }, }, }, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts index 887017ccaf..365fce3bf0 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts @@ -1,19 +1,20 @@ -import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; -import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; - -import { initializeAgentExecutorWithOptions } from 'langchain/agents'; import type { BaseChatMemory } from '@langchain/community/memory/chat_memory'; import type { BaseOutputParser } from '@langchain/core/output_parsers'; import { PromptTemplate } from '@langchain/core/prompts'; +import { initializeAgentExecutorWithOptions } from 'langchain/agents'; import { CombiningOutputParser } from 'langchain/output_parsers'; +import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; +import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; + import { isChatInstance, getPromptInputByType, - getOptionalOutputParsers, getConnectedTools, } from '../../../../../utils/helpers'; -import { getTracingConfig } from '../../../../../utils/tracing'; +import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser'; import { throwIfToolSchema } from '../../../../../utils/schemaParsing'; +import { getTracingConfig } from '../../../../../utils/tracing'; +import { extractParsedOutput } from '../utils'; export async function conversationalAgentExecute( this: IExecuteFunctions, @@ -102,12 +103,12 @@ export async function conversationalAgentExecute( input = (await prompt.invoke({ input })).value; } - let response = await agentExecutor + const response = await agentExecutor .withConfig(getTracingConfig(this)) .invoke({ input, outputParsers }); if (outputParser) { - response = { output: await outputParser.parse(response.output as string) }; + response.output = await extractParsedOutput(this, outputParser, response.output as string); } returnData.push({ json: response }); diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts index 12e1dbda4e..a9b324678c 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts @@ -1,3 +1,10 @@ +import type { BaseOutputParser } from '@langchain/core/output_parsers'; +import { PromptTemplate } from '@langchain/core/prompts'; +import { ChatOpenAI } from '@langchain/openai'; +import type { AgentExecutorInput } from 'langchain/agents'; +import { AgentExecutor, OpenAIAgent } from 'langchain/agents'; +import { BufferMemory, type BaseChatMemory } from 'langchain/memory'; +import { CombiningOutputParser } from 'langchain/output_parsers'; import { type IExecuteFunctions, type INodeExecutionData, @@ -5,19 +12,10 @@ import { NodeOperationError, } from 'n8n-workflow'; -import type { AgentExecutorInput } from 'langchain/agents'; -import { AgentExecutor, OpenAIAgent } from 'langchain/agents'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; -import { PromptTemplate } from '@langchain/core/prompts'; -import { CombiningOutputParser } from 'langchain/output_parsers'; -import { BufferMemory, type BaseChatMemory } from 'langchain/memory'; -import { ChatOpenAI } from '@langchain/openai'; -import { - getConnectedTools, - getOptionalOutputParsers, - getPromptInputByType, -} from '../../../../../utils/helpers'; +import { getConnectedTools, getPromptInputByType } from '../../../../../utils/helpers'; +import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser'; import { getTracingConfig } from '../../../../../utils/tracing'; +import { extractParsedOutput } from '../utils'; export async function openAiFunctionsAgentExecute( this: IExecuteFunctions, @@ -106,12 +104,12 @@ export async function openAiFunctionsAgentExecute( input = (await prompt.invoke({ input })).value; } - let response = await agentExecutor + const response = await agentExecutor .withConfig(getTracingConfig(this)) .invoke({ input, outputParsers }); if (outputParser) { - response = { output: await outputParser.parse(response.output as string) }; + response.output = await extractParsedOutput(this, outputParser, response.output as string); } returnData.push({ json: response }); diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts index a4ae1a0f1c..9425caaf84 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts @@ -1,3 +1,8 @@ +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { BaseOutputParser } from '@langchain/core/output_parsers'; +import { PromptTemplate } from '@langchain/core/prompts'; +import { PlanAndExecuteAgentExecutor } from 'langchain/experimental/plan_and_execute'; +import { CombiningOutputParser } from 'langchain/output_parsers'; import { type IExecuteFunctions, type INodeExecutionData, @@ -5,18 +10,11 @@ import { NodeOperationError, } from 'n8n-workflow'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; -import { PromptTemplate } from '@langchain/core/prompts'; -import { CombiningOutputParser } from 'langchain/output_parsers'; -import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { PlanAndExecuteAgentExecutor } from 'langchain/experimental/plan_and_execute'; -import { - getConnectedTools, - getOptionalOutputParsers, - getPromptInputByType, -} from '../../../../../utils/helpers'; -import { getTracingConfig } from '../../../../../utils/tracing'; +import { getConnectedTools, getPromptInputByType } from '../../../../../utils/helpers'; +import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser'; import { throwIfToolSchema } from '../../../../../utils/schemaParsing'; +import { getTracingConfig } from '../../../../../utils/tracing'; +import { extractParsedOutput } from '../utils'; export async function planAndExecuteAgentExecute( this: IExecuteFunctions, @@ -82,12 +80,12 @@ export async function planAndExecuteAgentExecute( input = (await prompt.invoke({ input })).value; } - let response = await agentExecutor + const response = await agentExecutor .withConfig(getTracingConfig(this)) .invoke({ input, outputParsers }); if (outputParser) { - response = { output: await outputParser.parse(response.output as string) }; + response.output = await extractParsedOutput(this, outputParser, response.output as string); } returnData.push({ json: response }); diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts index 11a5acb040..429adfad21 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts @@ -1,3 +1,9 @@ +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { BaseOutputParser } from '@langchain/core/output_parsers'; +import { PromptTemplate } from '@langchain/core/prompts'; +import { AgentExecutor, ChatAgent, ZeroShotAgent } from 'langchain/agents'; +import { CombiningOutputParser } from 'langchain/output_parsers'; import { type IExecuteFunctions, type INodeExecutionData, @@ -5,20 +11,15 @@ import { NodeOperationError, } from 'n8n-workflow'; -import { AgentExecutor, ChatAgent, ZeroShotAgent } from 'langchain/agents'; -import type { BaseLanguageModel } from '@langchain/core/language_models/base'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; -import { PromptTemplate } from '@langchain/core/prompts'; -import { CombiningOutputParser } from 'langchain/output_parsers'; -import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { getConnectedTools, - getOptionalOutputParsers, getPromptInputByType, isChatInstance, } from '../../../../../utils/helpers'; -import { getTracingConfig } from '../../../../../utils/tracing'; +import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser'; import { throwIfToolSchema } from '../../../../../utils/schemaParsing'; +import { getTracingConfig } from '../../../../../utils/tracing'; +import { extractParsedOutput } from '../utils'; export async function reActAgentAgentExecute( this: IExecuteFunctions, @@ -103,12 +104,12 @@ export async function reActAgentAgentExecute( input = (await prompt.invoke({ input })).value; } - let response = await agentExecutor + const response = await agentExecutor .withConfig(getTracingConfig(this)) .invoke({ input, outputParsers }); if (outputParser) { - response = { output: await outputParser.parse(response.output as string) }; + response.output = await extractParsedOutput(this, outputParser, response.output as string); } returnData.push({ json: response }); diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts index 90952bac41..84d775d0f5 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts @@ -1,7 +1,6 @@ import type { BaseChatMemory } from '@langchain/community/memory/chat_memory'; import { HumanMessage } from '@langchain/core/messages'; import type { BaseMessage } from '@langchain/core/messages'; -import type { BaseOutputParser, StructuredOutputParser } from '@langchain/core/output_parsers'; import type { BaseMessagePromptTemplateLike } from '@langchain/core/prompts'; import { ChatPromptTemplate } from '@langchain/core/prompts'; import { RunnableSequence } from '@langchain/core/runnables'; @@ -9,7 +8,6 @@ import type { Tool } from '@langchain/core/tools'; import { DynamicStructuredTool } from '@langchain/core/tools'; import type { AgentAction, AgentFinish } from 'langchain/agents'; import { AgentExecutor, createToolCallingAgent } from 'langchain/agents'; -import { OutputFixingParser } from 'langchain/output_parsers'; import { omit } from 'lodash'; import { BINARY_ENCODING, jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; @@ -20,24 +18,16 @@ import { SYSTEM_MESSAGE } from './prompt'; import { isChatInstance, getPromptInputByType, - getOptionalOutputParsers, getConnectedTools, } from '../../../../../utils/helpers'; +import { + getOptionalOutputParsers, + type N8nOutputParser, +} from '../../../../../utils/output_parsers/N8nOutputParser'; -function getOutputParserSchema(outputParser: BaseOutputParser): ZodObject { - const parserType = outputParser.lc_namespace[outputParser.lc_namespace.length - 1]; - let schema: ZodObject; - - if (parserType === 'structured') { - // If the output parser is a structured output parser, we will use the schema from the parser - schema = (outputParser as StructuredOutputParser>).schema; - } else if (parserType === 'fix' && outputParser instanceof OutputFixingParser) { - // If the output parser is a fixing parser, we will use the schema from the connected structured output parser - schema = (outputParser.parser as StructuredOutputParser>).schema; - } else { - // If the output parser is not a structured output parser, we will use a fallback schema - schema = z.object({ text: z.string() }); - } +function getOutputParserSchema(outputParser: N8nOutputParser): ZodObject { + const schema = + (outputParser.getSchema() as ZodObject) ?? z.object({ text: z.string() }); return schema; } @@ -205,10 +195,9 @@ export async function toolsAgentExecute(this: IExecuteFunctions): Promise step.tool === 'format_final_response'); if (responseParserTool) { const toolInput = responseParserTool?.toolInput; - const returnValues = (await outputParser.parse(toolInput as unknown as string)) as Record< - string, - unknown - >; + // Check if the tool input is a string or an object and convert it to a string + const parserInput = toolInput instanceof Object ? JSON.stringify(toolInput) : toolInput; + const returnValues = (await outputParser.parse(parserInput)) as Record; return handleParsedStepOutput(returnValues); } @@ -294,14 +283,6 @@ export async function toolsAgentExecute(this: IExecuteFunctions): Promise, + output: string, +): Promise | undefined> { + const parsedOutput = (await outputParser.parse(output)) as { + output: Record; + }; + + if (ctx.getNode().typeVersion <= 1.6) { + return parsedOutput; + } + // For 1.7 and above, we try to extract the output from the parsed output + // with fallback to the original output if it's not present + return parsedOutput?.output ?? parsedOutput; +} diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts index 71022f84fe..a080862c0f 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts @@ -1,9 +1,17 @@ +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { HumanMessage } from '@langchain/core/messages'; import { - ApplicationError, - NodeApiError, - NodeConnectionType, - NodeOperationError, -} from 'n8n-workflow'; + AIMessagePromptTemplate, + PromptTemplate, + SystemMessagePromptTemplate, + HumanMessagePromptTemplate, + ChatPromptTemplate, +} from '@langchain/core/prompts'; +import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; +import { ChatOllama } from '@langchain/ollama'; +import { LLMChain } from 'langchain/chains'; +import { CombiningOutputParser } from 'langchain/output_parsers'; import type { IBinaryData, IDataObject, @@ -12,28 +20,17 @@ import type { INodeType, INodeTypeDescription, } from 'n8n-workflow'; +import { + ApplicationError, + NodeApiError, + NodeConnectionType, + NodeOperationError, +} from 'n8n-workflow'; -import type { BaseLanguageModel } from '@langchain/core/language_models/base'; -import { - AIMessagePromptTemplate, - PromptTemplate, - SystemMessagePromptTemplate, - HumanMessagePromptTemplate, - ChatPromptTemplate, -} from '@langchain/core/prompts'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; -import { CombiningOutputParser } from 'langchain/output_parsers'; -import { LLMChain } from 'langchain/chains'; -import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { HumanMessage } from '@langchain/core/messages'; -import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; -import { ChatOllama } from '@langchain/ollama'; +import { getPromptInputByType, isChatInstance } from '../../../utils/helpers'; +import type { N8nOutputParser } from '../../../utils/output_parsers/N8nOutputParser'; +import { getOptionalOutputParsers } from '../../../utils/output_parsers/N8nOutputParser'; import { getTemplateNoticeField } from '../../../utils/sharedFields'; -import { - getOptionalOutputParsers, - getPromptInputByType, - isChatInstance, -} from '../../../utils/helpers'; import { getTracingConfig } from '../../../utils/tracing'; import { getCustomErrorMessage as getCustomOpenAiErrorMessage, @@ -189,7 +186,7 @@ async function getChain( itemIndex: number, query: string, llm: BaseLanguageModel, - outputParsers: BaseOutputParser[], + outputParsers: N8nOutputParser[], messages?: MessagesTemplate[], ): Promise { const chatTemplate: ChatPromptTemplate | PromptTemplate = await getChainPromptTemplate( diff --git a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts index 7ccfddc5e4..ab6cd8f201 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts @@ -1,3 +1,8 @@ +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { HumanMessage } from '@langchain/core/messages'; +import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts'; +import type { JSONSchema7 } from 'json-schema'; +import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers'; import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import type { INodeType, @@ -6,21 +11,17 @@ import type { INodeExecutionData, INodePropertyOptions, } from 'n8n-workflow'; -import type { JSONSchema7 } from 'json-schema'; -import type { BaseLanguageModel } from '@langchain/core/language_models/base'; -import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts'; import type { z } from 'zod'; -import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers'; -import { HumanMessage } from '@langchain/core/messages'; -import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; + +import { makeZodSchemaFromAttributes } from './helpers'; +import type { AttributeDefinition } from './types'; import { inputSchemaField, jsonSchemaExampleField, schemaTypeField, } from '../../../utils/descriptions'; +import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing'; import { getTracingConfig } from '../../../utils/tracing'; -import type { AttributeDefinition } from './types'; -import { makeZodSchemaFromAttributes } from './helpers'; const SYSTEM_PROMPT_TEMPLATE = `You are an expert extraction algorithm. Only extract relevant information from the text. @@ -261,8 +262,7 @@ export class InformationExtractor implements INodeType { jsonSchema = jsonParse(inputSchema); } - const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); - const zodSchema = await zodSchemaSandbox.runCode>(); + const zodSchema = convertJsonSchemaToZod>(jsonSchema); parser = OutputFixingParser.fromLLM(llm, StructuredOutputParser.fromZodSchema(zodSchema)); } diff --git a/packages/@n8n/nodes-langchain/nodes/code/Code.node.ts b/packages/@n8n/nodes-langchain/nodes/code/Code.node.ts index abe9b01530..8708e0c7ea 100644 --- a/packages/@n8n/nodes-langchain/nodes/code/Code.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/code/Code.node.ts @@ -1,13 +1,13 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ -import { - NodeOperationError, - type IExecuteFunctions, - type INodeExecutionData, - type INodeType, - type INodeTypeDescription, - type INodeOutputConfiguration, - type SupplyData, - NodeConnectionType, +import { NodeOperationError, NodeConnectionType } from 'n8n-workflow'; +import type { + IExecuteFunctions, + INodeExecutionData, + INodeType, + INodeTypeDescription, + INodeOutputConfiguration, + SupplyData, + ISupplyDataFunctions, } from 'n8n-workflow'; // TODO: Add support for execute function. Got already started but got commented out @@ -72,7 +72,7 @@ export const vmResolver = makeResolverFromLegacyOptions({ }); function getSandbox( - this: IExecuteFunctions, + this: IExecuteFunctions | ISupplyDataFunctions, code: string, options?: { addItems?: boolean; itemIndex?: number }, ) { @@ -354,7 +354,7 @@ export class Code implements INodeType { } } - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const code = this.getNodeParameter('code', itemIndex) as { supplyData?: { code: string } }; if (!code.supplyData?.code) { diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts index 783f12be9d..2e68db4e69 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -177,7 +177,7 @@ export class DocumentBinaryInputLoader implements INodeType { ], }; - async supplyData(this: IExecuteFunctions): Promise { + async supplyData(this: ISupplyDataFunctions): Promise { this.logger.debug('Supply Data for Binary Input Loader'); const textSplitter = (await this.getInputConnectionData( NodeConnectionType.AiTextSplitter, diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts index 062008db2f..5e6457951e 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -283,7 +283,7 @@ export class DocumentDefaultDataLoader implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const dataType = this.getNodeParameter('dataType', itemIndex, 'json') as 'json' | 'binary'; const textSplitter = (await this.getInputConnectionData( NodeConnectionType.AiTextSplitter, diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts index 916f0e7159..071134f25e 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { GithubRepoLoader } from '@langchain/community/document_loaders/web/github'; @@ -93,7 +93,7 @@ export class DocumentGithubLoader implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { console.log('Supplying data for Github Document Loader'); const repository = this.getNodeParameter('repository', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts index 3cb2c4bfdb..2e8cb95a11 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -79,7 +79,7 @@ export class DocumentJsonInputLoader implements INodeType { ], }; - async supplyData(this: IExecuteFunctions): Promise { + async supplyData(this: ISupplyDataFunctions): Promise { this.logger.debug('Supply Data for JSON Input Loader'); const textSplitter = (await this.getInputConnectionData( NodeConnectionType.AiTextSplitter, diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAwsBedrock/EmbeddingsAwsBedrock.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAwsBedrock/EmbeddingsAwsBedrock.node.ts index 58caebe05b..6e0782f1c1 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAwsBedrock/EmbeddingsAwsBedrock.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAwsBedrock/EmbeddingsAwsBedrock.node.ts @@ -2,9 +2,9 @@ import { BedrockEmbeddings } from '@langchain/aws'; import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -104,7 +104,7 @@ export class EmbeddingsAwsBedrock implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('aws'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts index 46195be0d3..a75a93c9f4 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -92,7 +92,7 @@ export class EmbeddingsAzureOpenAi implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings'); const credentials = await this.getCredentials<{ apiKey: string; diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts index a6c246acb5..26e5d39b70 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { CohereEmbeddings } from '@langchain/cohere'; @@ -99,7 +99,7 @@ export class EmbeddingsCohere implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings Cohere'); const modelName = this.getNodeParameter('modelName', itemIndex, 'embed-english-v2.0') as string; const credentials = await this.getCredentials<{ apiKey: string }>('cohereApi'); diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts index 92882dfffa..2a455e4574 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai'; @@ -116,7 +116,7 @@ export class EmbeddingsGoogleGemini implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings Google Gemini'); const modelName = this.getNodeParameter( 'modelName', diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts index 93d751b9c4..c8317630c3 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { HuggingFaceInferenceEmbeddings } from '@langchain/community/embeddings/hf'; @@ -81,7 +81,7 @@ export class EmbeddingsHuggingFaceInference implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings HF Inference'); const model = this.getNodeParameter( 'modelName', diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsMistralCloud/EmbeddingsMistralCloud.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsMistralCloud/EmbeddingsMistralCloud.node.ts index d8223a2ffe..dbfb93b82e 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsMistralCloud/EmbeddingsMistralCloud.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsMistralCloud/EmbeddingsMistralCloud.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { MistralAIEmbeddingsParams } from '@langchain/mistralai'; @@ -134,7 +134,7 @@ export class EmbeddingsMistralCloud implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('mistralCloudApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; const options = this.getNodeParameter( diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts index ec404f2306..d84aa537ec 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { OllamaEmbeddings } from '@langchain/ollama'; @@ -44,7 +44,7 @@ export class EmbeddingsOllama implements INodeType { properties: [getConnectionHintNoticeField([NodeConnectionType.AiVectorStore]), ollamaModel], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings Ollama'); const modelName = this.getNodeParameter('model', itemIndex) as string; const credentials = await this.getCredentials('ollamaApi'); diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts index 046f3e4f56..167581ed2e 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts @@ -1,10 +1,10 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, type SupplyData, + type ISupplyDataFunctions, type INodeProperties, } from 'n8n-workflow'; @@ -79,7 +79,7 @@ export class EmbeddingsOpenAi implements INodeType { }, ], group: ['transform'], - version: 1, + version: [1, 1.1], description: 'Use Embeddings OpenAI', defaults: { name: 'Embeddings OpenAI', @@ -170,7 +170,7 @@ export class EmbeddingsOpenAi implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply data for embeddings'); const credentials = await this.getCredentials('openAiApi'); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts index ecc14e1344..416a28b655 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts @@ -3,7 +3,7 @@ import { NodeConnectionType, type INodePropertyOptions, type INodeProperties, - type IExecuteFunctions, + type ISupplyDataFunctions, type INodeType, type INodeTypeDescription, type SupplyData, @@ -175,7 +175,7 @@ export class LmChatAnthropic implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('anthropicApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts index b4fc474dd2..dc2f716b2b 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -52,7 +52,7 @@ export class LmChatOllama implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('ollamaApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts index dcf483a751..2e724bf3a7 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, type JsonObject, NodeApiError, @@ -128,7 +128,7 @@ export class LmChatOpenAi implements INodeType { property: 'model', }, }, - default: 'gpt-3.5-turbo', + default: 'gpt-4o-mini', }, { displayName: @@ -242,7 +242,7 @@ export class LmChatOpenAi implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('openAiApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts index 191209bb33..6957cd9d9a 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -90,7 +90,7 @@ export class LmCohere implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('cohereApi'); const options = this.getNodeParameter('options', itemIndex, {}) as object; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts index 5492a51a97..f71708cbca 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -51,7 +51,7 @@ export class LmOllama implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('ollamaApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/description.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/description.ts index 382de60fdd..f91c9a1148 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/description.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/description.ts @@ -17,7 +17,7 @@ export const ollamaModel: INodeProperties = { displayName: 'Model', name: 'model', type: 'options', - default: 'llama2', + default: 'llama3.2', description: 'The model which will generate the completion. To download models, visit Ollama Models Library.', typeOptions: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts index a46ad429a2..5fb9be937e 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType } from 'n8n-workflow'; import type { - IExecuteFunctions, INodeType, INodeTypeDescription, + ISupplyDataFunctions, SupplyData, ILoadOptionsFunctions, } from 'n8n-workflow'; @@ -229,7 +229,7 @@ export class LmOpenAi implements INodeType { }, }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('openAiApi'); const modelName = this.getNodeParameter('model', itemIndex, '', { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts index 7b2c821f9c..ddf8065bf6 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -132,7 +132,7 @@ export class LmOpenHuggingFaceInference implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('huggingFaceApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts index c7b3d8ad95..b4eafde76e 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts @@ -2,9 +2,9 @@ import { ChatBedrockConverse } from '@langchain/aws'; import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -132,7 +132,7 @@ export class LmChatAwsBedrock implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('aws'); const modelName = this.getNodeParameter('model', itemIndex) as string; const options = this.getNodeParameter('options', itemIndex, {}) as { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts index 03548142db..55a5afb7ce 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -162,7 +162,7 @@ export class LmChatAzureOpenAi implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials<{ apiKey: string; resourceName: string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts index ce08a650f2..44691a47ef 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; @@ -113,7 +113,7 @@ export class LmChatGoogleGemini implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('googlePalmApi'); const modelName = this.getNodeParameter('modelName', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts index 55ccda90d2..a9a01ebf1b 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, type ILoadOptionsFunctions, type JsonObject, @@ -124,7 +124,7 @@ export class LmChatGoogleVertex implements INodeType { }, }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('googleApi'); const privateKey = formatPrivateKey(credentials.privateKey as string); const email = (credentials.email as string).trim(); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts index d0a28715e1..3588cf0cc3 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -129,7 +129,7 @@ export class LmChatGroq implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('groqApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts index 129beeadfe..5ff28bd30d 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -172,7 +172,7 @@ export class LmChatMistralCloud implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('mistralCloudApi'); const modelName = this.getNodeParameter('model', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts b/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts index 7d9049a037..660bf3b0a9 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts @@ -7,7 +7,7 @@ import type { SerializedSecret, } from '@langchain/core/load/serializable'; import type { LLMResult } from '@langchain/core/outputs'; -import type { IDataObject, IExecuteFunctions } from 'n8n-workflow'; +import type { IDataObject, ISupplyDataFunctions } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; import { pick } from 'lodash'; import type { BaseMessage } from '@langchain/core/messages'; @@ -26,12 +26,10 @@ type RunDetail = { options: SerializedSecret | SerializedNotImplemented | SerializedFields; }; -const TIKTOKEN_ESTIMATE_MODEL = 'gpt-3.5-turbo'; +const TIKTOKEN_ESTIMATE_MODEL = 'gpt-4o'; export class N8nLlmTracing extends BaseCallbackHandler { name = 'N8nLlmTracing'; - executionFunctions: IExecuteFunctions; - connectionType = NodeConnectionType.AiLanguageModel; promptTokensEstimate = 0; @@ -61,11 +59,10 @@ export class N8nLlmTracing extends BaseCallbackHandler { }; constructor( - executionFunctions: IExecuteFunctions, + private executionFunctions: ISupplyDataFunctions, options?: { tokensUsageParser: TokensUsageParser }, ) { super(); - this.executionFunctions = executionFunctions; this.options = { ...this.options, ...options }; } @@ -138,7 +135,7 @@ export class N8nLlmTracing extends BaseCallbackHandler { this.executionFunctions.addOutputData(this.connectionType, runDetails.index, [ [{ json: { ...response } }], ]); - void logAiEvent(this.executionFunctions, 'ai-llm-generated-output', { + logAiEvent(this.executionFunctions, 'ai-llm-generated-output', { messages: parsedMessages, options: runDetails.options, response, @@ -186,7 +183,7 @@ export class N8nLlmTracing extends BaseCallbackHandler { }); } - void logAiEvent(this.executionFunctions, 'ai-llm-errored', { + logAiEvent(this.executionFunctions, 'ai-llm-errored', { error: Object.keys(error).length === 0 ? error.toString() : error, runId, parentRunId, diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts index b8eea7a5e2..fae6927c25 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { BufferWindowMemoryInput } from 'langchain/memory'; @@ -134,7 +134,7 @@ export class MemoryBufferWindow implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const contextWindowLength = this.getNodeParameter('contextWindowLength', itemIndex) as number; const workflowId = this.getWorkflow().id; const memoryInstance = MemoryChatBufferSingleton.getInstance(); diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryMotorhead/MemoryMotorhead.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryMotorhead/MemoryMotorhead.node.ts index 9b2f46fb30..7f8d88fbaf 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryMotorhead/MemoryMotorhead.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryMotorhead/MemoryMotorhead.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -86,7 +86,7 @@ export class MemoryMotorhead implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('motorheadApi'); const nodeVersion = this.getNode().typeVersion; diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts index f42cb93fe1..f51b76fb18 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts @@ -1,5 +1,10 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ -import type { IExecuteFunctions, INodeType, INodeTypeDescription, SupplyData } from 'n8n-workflow'; +import type { + ISupplyDataFunctions, + INodeType, + INodeTypeDescription, + SupplyData, +} from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; import { PostgresChatMessageHistory } from '@langchain/community/stores/message/postgres'; @@ -73,7 +78,7 @@ export class MemoryPostgresChat implements INodeType { }, }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('postgres'); const tableName = this.getNodeParameter('tableName', itemIndex, 'n8n_chat_histories') as string; const sessionId = getSessionId(this, itemIndex); diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts index da57ede1d2..01a31458de 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeOperationError, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, NodeConnectionType, } from 'n8n-workflow'; @@ -102,7 +102,7 @@ export class MemoryRedisChat implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('redis'); const nodeVersion = this.getNode().typeVersion; diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts index f0177d9e75..be431b9b3c 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts @@ -1,6 +1,11 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; -import type { IExecuteFunctions, INodeType, INodeTypeDescription, SupplyData } from 'n8n-workflow'; +import type { + ISupplyDataFunctions, + INodeType, + INodeTypeDescription, + SupplyData, +} from 'n8n-workflow'; import { XataChatMessageHistory } from '@langchain/community/stores/message/xata'; import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; import { BaseClient } from '@xata.io/client'; @@ -88,7 +93,7 @@ export class MemoryXata implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('xataApi'); const nodeVersion = this.getNode().typeVersion; diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryZep/MemoryZep.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryZep/MemoryZep.node.ts index 7cbf1da574..20e70fd920 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryZep/MemoryZep.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryZep/MemoryZep.node.ts @@ -1,7 +1,7 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, + type ISupplyDataFunctions, type INodeType, type INodeTypeDescription, type SupplyData, @@ -103,7 +103,7 @@ export class MemoryZep implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials<{ apiKey?: string; apiUrl?: string; diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts index 97c86506b7..d4743fb043 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts @@ -1,15 +1,16 @@ -/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ -import { - NodeConnectionType, - type IExecuteFunctions, - type INodeType, - type INodeTypeDescription, - type SupplyData, -} from 'n8n-workflow'; -import { OutputFixingParser } from 'langchain/output_parsers'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; import type { BaseLanguageModel } from '@langchain/core/language_models/base'; -import { logWrapper } from '../../../utils/logWrapper'; +import { NodeConnectionType } from 'n8n-workflow'; +import type { + ISupplyDataFunctions, + INodeType, + INodeTypeDescription, + SupplyData, +} from 'n8n-workflow'; + +import { + N8nOutputFixingParser, + type N8nStructuredOutputParser, +} from '../../../utils/output_parsers/N8nOutputParser'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; export class OutputParserAutofixing implements INodeType { @@ -67,7 +68,7 @@ export class OutputParserAutofixing implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, itemIndex, @@ -75,12 +76,12 @@ export class OutputParserAutofixing implements INodeType { const outputParser = (await this.getInputConnectionData( NodeConnectionType.AiOutputParser, itemIndex, - )) as BaseOutputParser; + )) as N8nStructuredOutputParser; - const parser = OutputFixingParser.fromLLM(model, outputParser); + const parser = new N8nOutputFixingParser(this, model, outputParser); return { - response: logWrapper(parser, this), + response: parser, }; } } diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/test/OutputParserAutofixing.node.test.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/test/OutputParserAutofixing.node.test.ts new file mode 100644 index 0000000000..32d25d4f73 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/test/OutputParserAutofixing.node.test.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import type { MockProxy } from 'jest-mock-extended'; +import { mock } from 'jest-mock-extended'; +import { normalizeItems } from 'n8n-core'; +import type { IExecuteFunctions, IWorkflowDataProxyData } from 'n8n-workflow'; +import { ApplicationError, NodeConnectionType } from 'n8n-workflow'; + +import { N8nOutputFixingParser } from '../../../../utils/output_parsers/N8nOutputParser'; +import type { N8nStructuredOutputParser } from '../../../../utils/output_parsers/N8nOutputParser'; +import { OutputParserAutofixing } from '../OutputParserAutofixing.node'; + +describe('OutputParserAutofixing', () => { + let outputParser: OutputParserAutofixing; + let thisArg: MockProxy; + let mockModel: MockProxy; + let mockStructuredOutputParser: MockProxy; + + beforeEach(() => { + outputParser = new OutputParserAutofixing(); + thisArg = mock({ + helpers: { normalizeItems }, + }); + mockModel = mock(); + mockStructuredOutputParser = mock(); + + thisArg.getWorkflowDataProxy.mockReturnValue(mock({ $input: mock() })); + thisArg.addInputData.mockReturnValue({ index: 0 }); + thisArg.addOutputData.mockReturnValue(); + thisArg.getInputConnectionData.mockImplementation(async (type: NodeConnectionType) => { + if (type === NodeConnectionType.AiLanguageModel) return mockModel; + if (type === NodeConnectionType.AiOutputParser) return mockStructuredOutputParser; + + throw new ApplicationError('Unexpected connection type'); + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + function getMockedRetryChain(output: string) { + return jest.fn().mockReturnValue({ + invoke: jest.fn().mockResolvedValue({ + content: output, + }), + }); + } + + it('should successfully parse valid output without needing to fix it', async () => { + const validOutput = { name: 'Alice', age: 25 }; + + mockStructuredOutputParser.parse.mockResolvedValueOnce(validOutput); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + // Ensure the response contains the output-fixing parser + expect(response).toBeDefined(); + expect(response).toBeInstanceOf(N8nOutputFixingParser); + + const result = await response.parse('{"name": "Alice", "age": 25}'); + + // Validate that the parser succeeds without retry + expect(result).toEqual(validOutput); + expect(mockStructuredOutputParser.parse).toHaveBeenCalledTimes(1); // Only one call to parse + }); + + it('should throw an error when both structured parser and fixing parser fail', async () => { + mockStructuredOutputParser.parse + .mockRejectedValueOnce(new Error('Invalid JSON')) // First attempt fails + .mockRejectedValueOnce(new Error('Fixing attempt failed')); // Second attempt fails + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = getMockedRetryChain('{}'); + + await expect(response.parse('Invalid JSON string')).rejects.toThrow('Fixing attempt failed'); + expect(mockStructuredOutputParser.parse).toHaveBeenCalledTimes(2); + }); + + it('should reject on the first attempt and succeed on retry with the parsed content', async () => { + const validOutput = { name: 'Bob', age: 28 }; + + mockStructuredOutputParser.parse.mockRejectedValueOnce(new Error('Invalid JSON')); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = getMockedRetryChain(JSON.stringify(validOutput)); + + mockStructuredOutputParser.parse.mockResolvedValueOnce(validOutput); + + const result = await response.parse('Invalid JSON string'); + + expect(result).toEqual(validOutput); + expect(mockStructuredOutputParser.parse).toHaveBeenCalledTimes(2); // First fails, second succeeds + }); + + it('should handle non-JSON formatted response from fixing parser', async () => { + mockStructuredOutputParser.parse.mockRejectedValueOnce(new Error('Invalid JSON')); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = getMockedRetryChain('This is not JSON'); + + mockStructuredOutputParser.parse.mockRejectedValueOnce(new Error('Unexpected token')); + + // Expect the structured parser to throw an error on invalid JSON from retry + await expect(response.parse('Invalid JSON string')).rejects.toThrow('Unexpected token'); + expect(mockStructuredOutputParser.parse).toHaveBeenCalledTimes(2); // First fails, second tries and fails + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/OutputParserItemList.node.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/OutputParserItemList.node.ts index 24327b2970..b613c14775 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/OutputParserItemList.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/OutputParserItemList.node.ts @@ -1,14 +1,14 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; -import { logWrapper } from '../../../utils/logWrapper'; + +import { N8nItemListOutputParser } from '../../../utils/output_parsers/N8nItemListOutputParser'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { ItemListOutputParser } from './ItemListOutputParser'; export class OutputParserItemList implements INodeType { description: INodeTypeDescription = { @@ -80,16 +80,16 @@ export class OutputParserItemList implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const options = this.getNodeParameter('options', itemIndex, {}) as { numberOfItems?: number; separator?: string; }; - const parser = new ItemListOutputParser(options); + const parser = new N8nItemListOutputParser(options); return { - response: logWrapper(parser, this), + response: parser, }; } } diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/test/OutputParserItemList.node.test.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/test/OutputParserItemList.node.test.ts new file mode 100644 index 0000000000..31e96077c4 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/test/OutputParserItemList.node.test.ts @@ -0,0 +1,123 @@ +import { mock } from 'jest-mock-extended'; +import { normalizeItems } from 'n8n-core'; +import { + ApplicationError, + type IExecuteFunctions, + type IWorkflowDataProxyData, +} from 'n8n-workflow'; + +import { N8nItemListOutputParser } from '../../../../utils/output_parsers/N8nItemListOutputParser'; +import { OutputParserItemList } from '../OutputParserItemList.node'; + +describe('OutputParserItemList', () => { + let outputParser: OutputParserItemList; + const thisArg = mock({ + helpers: { normalizeItems }, + }); + const workflowDataProxy = mock({ $input: mock() }); + + beforeEach(() => { + outputParser = new OutputParserItemList(); + thisArg.getWorkflowDataProxy.mockReturnValue(workflowDataProxy); + thisArg.addInputData.mockReturnValue({ index: 0 }); + thisArg.addOutputData.mockReturnValue(); + thisArg.getNodeParameter.mockReset(); + }); + + describe('supplyData', () => { + it('should create a parser with default options', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return {}; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + expect(response).toBeInstanceOf(N8nItemListOutputParser); + }); + + it('should create a parser with custom number of items', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return { numberOfItems: 5 }; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + expect(response).toBeInstanceOf(N8nItemListOutputParser); + expect((response as any).numberOfItems).toBe(5); + }); + + it('should create a parser with custom separator', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return { separator: ',' }; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + expect(response).toBeInstanceOf(N8nItemListOutputParser); + expect((response as any).separator).toBe(','); + }); + }); + + describe('parse', () => { + it('should parse a list with default separator', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return {}; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + const result = await (response as N8nItemListOutputParser).parse('item1\nitem2\nitem3'); + expect(result).toEqual(['item1', 'item2', 'item3']); + }); + + it('should parse a list with custom separator', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return { separator: ',' }; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + const result = await (response as N8nItemListOutputParser).parse('item1,item2,item3'); + expect(result).toEqual(['item1', 'item2', 'item3']); + }); + + it('should limit the number of items returned', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return { numberOfItems: 2 }; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + const result = await (response as N8nItemListOutputParser).parse( + 'item1\nitem2\nitem3\nitem4', + ); + expect(result).toEqual(['item1', 'item2']); + }); + + it('should throw an error if not enough items are returned', async () => { + thisArg.getNodeParameter.mockImplementation((parameterName) => { + if (parameterName === 'options') { + return { numberOfItems: 5 }; + } + throw new ApplicationError('Not implemented'); + }); + + const { response } = await outputParser.supplyData.call(thisArg, 0); + await expect( + (response as N8nItemListOutputParser).parse('item1\nitem2\nitem3'), + ).rejects.toThrow('Wrong number of items returned'); + }); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts index 6ce6bff76b..c35cb1d145 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts @@ -1,90 +1,24 @@ -/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ +import type { JSONSchema7 } from 'json-schema'; import { jsonParse, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, NodeOperationError, NodeConnectionType, } from 'n8n-workflow'; -import { z } from 'zod'; -import type { JSONSchema7 } from 'json-schema'; -import { StructuredOutputParser } from 'langchain/output_parsers'; -import { OutputParserException } from '@langchain/core/output_parsers'; -import get from 'lodash/get'; -import type { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox'; -import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { logWrapper } from '../../../utils/logWrapper'; -import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; +import type { z } from 'zod'; + import { inputSchemaField, jsonSchemaExampleField, schemaTypeField, } from '../../../utils/descriptions'; +import { N8nStructuredOutputParser } from '../../../utils/output_parsers/N8nOutputParser'; +import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing'; +import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -const STRUCTURED_OUTPUT_KEY = '__structured__output'; -const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object'; -const STRUCTURED_OUTPUT_ARRAY_KEY = '__structured__output__array'; - -export class N8nStructuredOutputParser extends StructuredOutputParser { - async parse(text: string): Promise> { - try { - const parsed = (await super.parse(text)) as object; - - return ( - get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_OBJECT_KEY]) ?? - get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_ARRAY_KEY]) ?? - get(parsed, STRUCTURED_OUTPUT_KEY) ?? - parsed - ); - } catch (e) { - // eslint-disable-next-line n8n-nodes-base/node-execute-block-wrong-error-thrown - throw new OutputParserException(`Failed to parse. Text: "${text}". Error: ${e}`, text); - } - } - - static async fromZedJsonSchema( - sandboxedSchema: JavaScriptSandbox, - nodeVersion: number, - ): Promise>> { - const zodSchema = await sandboxedSchema.runCode>(); - - let returnSchema: z.ZodSchema; - if (nodeVersion === 1) { - returnSchema = z.object({ - [STRUCTURED_OUTPUT_KEY]: z - .object({ - [STRUCTURED_OUTPUT_OBJECT_KEY]: zodSchema.optional(), - [STRUCTURED_OUTPUT_ARRAY_KEY]: z.array(zodSchema).optional(), - }) - .describe( - `Wrapper around the output data. It can only contain ${STRUCTURED_OUTPUT_OBJECT_KEY} or ${STRUCTURED_OUTPUT_ARRAY_KEY} but never both.`, - ) - .refine( - (data) => { - // Validate that one and only one of the properties exists - return ( - Boolean(data[STRUCTURED_OUTPUT_OBJECT_KEY]) !== - Boolean(data[STRUCTURED_OUTPUT_ARRAY_KEY]) - ); - }, - { - message: - 'One and only one of __structured__output__object and __structured__output__array should be present.', - path: [STRUCTURED_OUTPUT_KEY], - }, - ), - }); - } else { - returnSchema = z.object({ - output: zodSchema.optional(), - }); - } - - return N8nStructuredOutputParser.fromZodSchema(returnSchema); - } -} export class OutputParserStructured implements INodeType { description: INodeTypeDescription = { displayName: 'Structured Output Parser', @@ -188,7 +122,7 @@ export class OutputParserStructured implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const schemaType = this.getNodeParameter('schemaType', itemIndex, '') as 'fromJson' | 'manual'; // We initialize these even though one of them will always be empty // it makes it easer to navigate the ternary operator @@ -204,15 +138,16 @@ export class OutputParserStructured implements INodeType { const jsonSchema = schemaType === 'fromJson' ? generateSchema(jsonExample) : jsonParse(inputSchema); - const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); + const zodSchema = convertJsonSchemaToZod>(jsonSchema); const nodeVersion = this.getNode().typeVersion; try { - const parser = await N8nStructuredOutputParser.fromZedJsonSchema( - zodSchemaSandbox, + const parser = await N8nStructuredOutputParser.fromZodJsonSchema( + zodSchema, nodeVersion, + this, ); return { - response: logWrapper(parser, this), + response: parser, }; } catch (error) { throw new NodeOperationError(this.getNode(), 'Error during parsing of JSON Schema.'); diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts index b4dd6708eb..af72c49d7e 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts @@ -1,8 +1,13 @@ -import type { IExecuteFunctions, INode, IWorkflowDataProxyData } from 'n8n-workflow'; import { mock } from 'jest-mock-extended'; import { normalizeItems } from 'n8n-core'; -import type { z } from 'zod'; -import type { StructuredOutputParser } from 'langchain/output_parsers'; +import { + jsonParse, + type IExecuteFunctions, + type INode, + type IWorkflowDataProxyData, +} from 'n8n-workflow'; + +import type { N8nStructuredOutputParser } from '../../../../utils/output_parsers/N8nStructuredOutputParser'; import { OutputParserStructured } from '../OutputParserStructured.node'; describe('OutputParserStructured', () => { @@ -11,139 +16,451 @@ describe('OutputParserStructured', () => { helpers: { normalizeItems }, }); const workflowDataProxy = mock({ $input: mock() }); - thisArg.getWorkflowDataProxy.mockReturnValue(workflowDataProxy); - thisArg.getNode.mockReturnValue(mock({ typeVersion: 1.1 })); - thisArg.addInputData.mockReturnValue({ index: 0 }); - thisArg.addOutputData.mockReturnValue(); beforeEach(() => { outputParser = new OutputParserStructured(); + thisArg.getWorkflowDataProxy.mockReturnValue(workflowDataProxy); + thisArg.addInputData.mockReturnValue({ index: 0 }); + thisArg.addOutputData.mockReturnValue(); }); describe('supplyData', () => { - it('should parse a valid JSON schema', async () => { - const schema = `{ - "type": "object", - "properties": { - "name": { - "type": "string" + describe('Version 1.1 and below', () => { + beforeEach(() => { + thisArg.getNode.mockReturnValue(mock({ typeVersion: 1.1 })); + }); + + it('should parse a complex nested schema', async () => { + const schema = `{ + "type": "object", + "properties": { + "user": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "details": { + "type": "object", + "properties": { + "age": { "type": "number" }, + "hobbies": { "type": "array", "items": { "type": "string" } } + } + } + } + }, + "timestamp": { "type": "string", "format": "date-time" } }, - "age": { - "type": "number" - } - }, - "required": ["name", "age"] - }`; - thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); - const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { - response: StructuredOutputParser>; - }; - const outputObject = { output: { name: 'Mac', age: 27 } }; - const parsersOutput = await response.parse(`Here's the output! - \`\`\`json - ${JSON.stringify(outputObject)} - \`\`\` - `); - - expect(parsersOutput).toEqual(outputObject); - }); - it('should handle missing required properties', async () => { - const schema = `{ - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "age": { - "type": "number" - } - }, - "required": ["name", "age"] - }`; - thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); - const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { - response: StructuredOutputParser>; - }; - const outputObject = { output: { name: 'Mac' } }; - - await expect( - response.parse(`Here's the output! - \`\`\`json - ${JSON.stringify(outputObject)} - \`\`\` - `), - ).rejects.toThrow('Required'); - }); - - it('should throw on wrong type', async () => { - const schema = `{ - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "age": { - "type": "number" - } - }, - "required": ["name", "age"] - }`; - thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); - const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { - response: StructuredOutputParser>; - }; - const outputObject = { output: { name: 'Mac', age: '27' } }; - - await expect( - response.parse(`Here's the output! - \`\`\`json - ${JSON.stringify(outputObject)} - \`\`\` - `), - ).rejects.toThrow('Expected number, received string'); - }); - - it('should parse array output', async () => { - const schema = `{ - "type": "object", - "properties": { - "myArr": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "age": { - "type": "number" - } + "required": ["user", "timestamp"] + }`; + thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + user: { + name: 'Alice', + details: { + age: 30, + hobbies: ['reading', 'hiking'], }, - "required": ["name", "age"] - } - } - }, - "required": ["myArr"] - }`; - thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); - const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { - response: StructuredOutputParser>; - }; - const outputObject = { - output: { - myArr: [ - { name: 'Mac', age: 27 }, - { name: 'Alice', age: 25 }, - ], - }, - }; - const parsersOutput = await response.parse(`Here's the output! - \`\`\`json - ${JSON.stringify(outputObject)} - \`\`\` - `); + }, + timestamp: '2023-04-01T12:00:00Z', + }, + }; + const parsersOutput = await response.parse(`Here's the complex output: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); - expect(parsersOutput).toEqual(outputObject); + expect(parsersOutput).toEqual(outputObject); + }); + + it('should handle optional fields correctly', async () => { + const schema = `{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "number" }, + "email": { "type": "string", "format": "email" } + }, + "required": ["name"] + }`; + thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + name: 'Bob', + email: 'bob@example.com', + }, + }; + const parsersOutput = await response.parse(`Here's the output with optional fields: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); + + it('should handle arrays of objects', async () => { + const schema = `{ + "type": "object", + "properties": { + "users": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "number" }, + "name": { "type": "string" } + }, + "required": ["id", "name"] + } + } + }, + "required": ["users"] + }`; + thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + users: [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + ], + }, + }; + const parsersOutput = await response.parse(`Here's the array output: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); + + it('should handle empty objects', async () => { + const schema = `{ + "type": "object", + "properties": { + "data": { + "type": "object" + } + }, + "required": ["data"] + }`; + thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + data: {}, + }, + }; + const parsersOutput = await response.parse(`Here's the empty object output: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); + + it('should throw error for null values in non-nullable fields', async () => { + const schema = `{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "number" } + }, + "required": ["name", "age"] + }`; + thisArg.getNodeParameter.calledWith('jsonSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + name: 'Charlie', + age: null, + }, + }; + + await expect( + response.parse( + `Here's the output with null value: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `, + undefined, + (e) => e, + ), + ).rejects.toThrow('Expected number, received null'); + }); + }); + + describe('Version 1.2 and above', () => { + beforeEach(() => { + thisArg.getNode.mockReturnValue(mock({ typeVersion: 1.2 })); + }); + + it('should parse output using schema generated from complex JSON example', async () => { + const jsonExample = `{ + "user": { + "name": "Alice", + "details": { + "age": 30, + "address": { + "street": "123 Main St", + "city": "Anytown", + "zipCode": "12345" + } + } + }, + "orders": [ + { + "id": "ORD-001", + "items": ["item1", "item2"], + "total": 50.99 + }, + { + "id": "ORD-002", + "items": ["item3"], + "total": 25.50 + } + ], + "isActive": true + }`; + thisArg.getNodeParameter.calledWith('schemaType', 0).mockReturnValueOnce('fromJson'); + thisArg.getNodeParameter + .calledWith('jsonSchemaExample', 0) + .mockReturnValueOnce(jsonExample); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + + const outputObject = { + output: jsonParse(jsonExample), + }; + + const parsersOutput = await response.parse(`Here's the complex output: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); + + it('should validate enum values', async () => { + const inputSchema = `{ + "type": "object", + "properties": { + "color": { + "type": "string", + "enum": ["red", "green", "blue"] + } + }, + "required": ["color"] + }`; + thisArg.getNodeParameter.calledWith('schemaType', 0).mockReturnValueOnce('manual'); + thisArg.getNodeParameter.calledWith('inputSchema', 0).mockReturnValueOnce(inputSchema); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + + const validOutput = { + output: { + color: 'green', + }, + }; + + const invalidOutput = { + output: { + color: 'yellow', + }, + }; + + await expect( + response.parse(`Valid output: + \`\`\`json + ${JSON.stringify(validOutput)} + \`\`\` + `), + ).resolves.toEqual(validOutput); + + await expect( + response.parse( + `Invalid output: + \`\`\`json + ${JSON.stringify(invalidOutput)} + \`\`\` + `, + undefined, + (e) => e, + ), + ).rejects.toThrow(); + }); + + it('should handle recursive structures', async () => { + const inputSchema = `{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#" } + } + }, + "required": ["name"] + }`; + thisArg.getNodeParameter.calledWith('schemaType', 0).mockReturnValueOnce('manual'); + thisArg.getNodeParameter.calledWith('inputSchema', 0).mockReturnValueOnce(inputSchema); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + + const outputObject = { + output: { + name: 'Root', + children: [ + { + name: 'Child1', + children: [{ name: 'Grandchild1' }, { name: 'Grandchild2' }], + }, + { + name: 'Child2', + }, + ], + }, + }; + + const parsersOutput = await response.parse(`Here's the recursive structure output: + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); + + it('should handle missing required properties', async () => { + const schema = `{ + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "age": { + "type": "number" + } + }, + "required": ["name", "age"] + }`; + thisArg.getNodeParameter.calledWith('schemaType', 0).mockReturnValueOnce('manual'); + thisArg.getNodeParameter.calledWith('inputSchema', 0).mockReturnValueOnce(schema); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { output: { name: 'Mac' } }; + + await expect( + response.parse( + `Here's the output! + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `, + undefined, + (e) => e, + ), + ).rejects.toThrow('Required'); + }); + it('should throw on wrong type', async () => { + const schema = `{ + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "age": { + "type": "number" + } + }, + "required": ["name", "age"] + }`; + thisArg.getNodeParameter.calledWith('inputSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { output: { name: 'Mac', age: '27' } }; + + await expect( + response.parse( + `Here's the output! + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `, + undefined, + (e) => e, + ), + ).rejects.toThrow('Expected number, received string'); + }); + + it('should parse array output', async () => { + const schema = `{ + "type": "object", + "properties": { + "myArr": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "age": { + "type": "number" + } + }, + "required": ["name", "age"] + } + } + }, + "required": ["myArr"] + }`; + thisArg.getNodeParameter.calledWith('inputSchema', 0).mockReturnValueOnce(schema); + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nStructuredOutputParser; + }; + const outputObject = { + output: { + myArr: [ + { name: 'Mac', age: 27 }, + { name: 'Alice', age: 25 }, + ], + }, + }; + const parsersOutput = await response.parse(`Here's the output! + \`\`\`json + ${JSON.stringify(outputObject)} + \`\`\` + `); + + expect(parsersOutput).toEqual(outputObject); + }); }); }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts index 5b89a0bf26..8017caa1ad 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -63,7 +63,7 @@ export class RetrieverContextualCompression implements INodeType { properties: [], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supplying data for Contextual Compression Retriever'); const model = (await this.getInputConnectionData( diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts index 3cb377d654..f814ba875e 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; @@ -82,7 +82,7 @@ export class RetrieverMultiQuery implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supplying data for MultiQuery Retriever'); const options = this.getNodeParameter('options', itemIndex, {}) as { queryCount?: number }; diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts index 6543d061d4..5e79a6a754 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { VectorStore } from '@langchain/core/vectorstores'; @@ -56,7 +56,7 @@ export class RetrieverVectorStore implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supplying data for Vector Store Retriever'); const topK = this.getNodeParameter('topK', itemIndex, 4) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverWorkflow/RetrieverWorkflow.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverWorkflow/RetrieverWorkflow.node.ts index 6b446149fc..0aafabf1d4 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverWorkflow/RetrieverWorkflow.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverWorkflow/RetrieverWorkflow.node.ts @@ -5,7 +5,7 @@ import type { IExecuteWorkflowInfo, INodeExecutionData, IWorkflowBase, - IExecuteFunctions, + ISupplyDataFunctions, INodeType, INodeTypeDescription, SupplyData, @@ -292,15 +292,15 @@ export class RetrieverWorkflow implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { class WorkflowRetriever extends BaseRetriever { lc_namespace = ['n8n-nodes-langchain', 'retrievers', 'workflow']; - executeFunctions: IExecuteFunctions; - - constructor(executeFunctions: IExecuteFunctions, fields: BaseRetrieverInput) { + constructor( + private executeFunctions: ISupplyDataFunctions, + fields: BaseRetrieverInput, + ) { super(fields); - this.executeFunctions = executeFunctions; } async _getRelevantDocuments( diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts index 61e62def0f..f62e8f01f1 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { CharacterTextSplitterParams } from '@langchain/textsplitters'; @@ -63,7 +63,7 @@ export class TextSplitterCharacterTextSplitter implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply Data for Text Splitter'); const separator = this.getNodeParameter('separator', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts index 4d2c5a6ec8..21a0520766 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { @@ -94,7 +94,7 @@ export class TextSplitterRecursiveCharacterTextSplitter implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply Data for Text Splitter'); const chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts index c021aa1df7..247d142fa8 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { TokenTextSplitter } from '@langchain/textsplitters'; @@ -56,7 +56,7 @@ export class TextSplitterTokenSplitter implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply Data for Text Splitter'); const chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolCalculator/ToolCalculator.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolCalculator/ToolCalculator.node.ts index f37a3176a4..f50a6216c0 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolCalculator/ToolCalculator.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolCalculator/ToolCalculator.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { Calculator } from '@langchain/community/tools/calculator'; @@ -43,7 +43,7 @@ export class ToolCalculator implements INodeType { properties: [getConnectionHintNoticeField([NodeConnectionType.AiAgent])], }; - async supplyData(this: IExecuteFunctions): Promise { + async supplyData(this: ISupplyDataFunctions): Promise { return { response: logWrapper(new Calculator(), this), }; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts index 7980f5fa9d..1491662e61 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts @@ -1,29 +1,28 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ +import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; +import type { JSONSchema7 } from 'json-schema'; +import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox'; +import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox'; +import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; +import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; import type { - IExecuteFunctions, INodeType, INodeTypeDescription, + ISupplyDataFunctions, SupplyData, ExecutionError, IDataObject, } from 'n8n-workflow'; - import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; -import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; -import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; -import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox'; -import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox'; -import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; -import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; +import type { DynamicZodObject } from '../../../types/zod.types'; import { inputSchemaField, jsonSchemaExampleField, schemaTypeField, } from '../../../utils/descriptions'; -import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; -import type { JSONSchema7 } from 'json-schema'; -import type { DynamicZodObject } from '../../../types/zod.types'; +import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing'; +import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; export class ToolCode implements INodeType { description: INodeTypeDescription = { @@ -176,7 +175,7 @@ export class ToolCode implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const node = this.getNode(); const workflowMode = this.getMode(); @@ -273,10 +272,9 @@ export class ToolCode implements INodeType { ? generateSchema(jsonExample) : jsonParse(inputSchema); - const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); - const zodSchema = await zodSchemaSandbox.runCode(); + const zodSchema = convertJsonSchemaToZod(jsonSchema); - tool = new DynamicStructuredTool({ + tool = new DynamicStructuredTool({ schema: zodSchema, ...commonToolOptions, }); diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts index 32f6be42e7..f279c1e751 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts @@ -1,8 +1,8 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import type { - IExecuteFunctions, INodeType, INodeTypeDescription, + ISupplyDataFunctions, SupplyData, IHttpRequestMethods, IHttpRequestOptions, @@ -250,7 +250,7 @@ export class ToolHttpRequest implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const name = this.getNode().name.replace(/ /g, '_'); try { tryToParseAlphanumericString(name); @@ -281,6 +281,8 @@ export class ToolHttpRequest implements INodeType { 'User-Agent': undefined, }, body: {}, + // We will need a full response object later to extract the headers and check the response's content type. + returnFullResponse: true, }; const authentication = this.getNodeParameter('authentication', itemIndex, 'none') as diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts new file mode 100644 index 0000000000..1a99896fff --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts @@ -0,0 +1,298 @@ +import { mock } from 'jest-mock-extended'; +import type { IExecuteFunctions, INode } from 'n8n-workflow'; +import { jsonParse } from 'n8n-workflow'; + +import type { N8nTool } from '../../../../utils/N8nTool'; +import { ToolHttpRequest } from '../ToolHttpRequest.node'; + +describe('ToolHttpRequest', () => { + const httpTool = new ToolHttpRequest(); + const helpers = mock(); + const executeFunctions = mock({ helpers }); + + beforeEach(() => { + jest.resetAllMocks(); + executeFunctions.getNode.mockReturnValue( + mock({ + type: 'n8n-nodes-base.httpRequest', + name: 'HTTP Request', + typeVersion: 1.1, + }), + ); + executeFunctions.addInputData.mockReturnValue({ index: 0 }); + }); + + describe('Binary response', () => { + it('should return the error when receiving a binary response', async () => { + helpers.httpRequest.mockResolvedValue({ + body: Buffer.from(''), + headers: { + 'content-type': 'image/jpeg', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/image/jpeg'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + expect(res).toContain('error'); + expect(res).toContain('Binary data is not supported'); + }); + + it('should return the response text when receiving a text response', async () => { + helpers.httpRequest.mockResolvedValue({ + body: 'Hello World', + headers: { + 'content-type': 'text/plain', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/text/plain'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + expect(res).toEqual('Hello World'); + }); + + it('should return the response text when receiving a text response with a charset', async () => { + helpers.httpRequest.mockResolvedValue({ + body: 'こんにちは世界', + headers: { + 'content-type': 'text/plain; charset=iso-2022-jp', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/text/plain'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + expect(res).toEqual('こんにちは世界'); + }); + + it('should return the response object when receiving a JSON response', async () => { + const mockJson = { hello: 'world' }; + + helpers.httpRequest.mockResolvedValue({ + body: JSON.stringify(mockJson), + headers: { + 'content-type': 'application/json', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/json'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + expect(jsonParse(res)).toEqual(mockJson); + }); + + it('should handle authentication with predefined credentials', async () => { + helpers.httpRequestWithAuthentication.mockResolvedValue({ + body: 'Hello World', + headers: { + 'content-type': 'text/plain', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/text/plain'; + case 'authentication': + return 'predefinedCredentialType'; + case 'nodeCredentialType': + return 'linearApi'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + + expect(res).toEqual('Hello World'); + + expect(helpers.httpRequestWithAuthentication).toHaveBeenCalledWith( + 'linearApi', + expect.objectContaining({ + returnFullResponse: true, + }), + undefined, + ); + }); + + it('should handle authentication with generic credentials', async () => { + helpers.httpRequest.mockResolvedValue({ + body: 'Hello World', + headers: { + 'content-type': 'text/plain', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/text/plain'; + case 'authentication': + return 'genericCredentialType'; + case 'genericAuthType': + return 'httpBasicAuth'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + executeFunctions.getCredentials.mockResolvedValue({ + user: 'username', + password: 'password', + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({}); + + expect(res).toEqual('Hello World'); + + expect(helpers.httpRequest).toHaveBeenCalledWith( + expect.objectContaining({ + returnFullResponse: true, + auth: expect.objectContaining({ + username: 'username', + password: 'password', + }), + }), + ); + }); + }); + + describe('Optimize response', () => { + it('should extract body from the response HTML', async () => { + helpers.httpRequest.mockResolvedValue({ + body: ` + + + + +

Test

+ +
+

+ Test content +

+
+ +`, + headers: { + 'content-type': 'text/html', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation( + (paramName: string, _: any, fallback: any) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return '{url}'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + case 'optimizeResponse': + return true; + case 'responseType': + return 'html'; + case 'cssSelector': + return 'body'; + default: + return fallback; + } + }, + ); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + + const res = await (response as N8nTool).invoke({ + url: 'https://httpbin.org/html', + }); + + expect(helpers.httpRequest).toHaveBeenCalled(); + expect(res).toEqual( + JSON.stringify(['

Test

Test content

'], null, 2), + ); + }); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts index c06a869a8d..f1d6dfd150 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts @@ -1,26 +1,23 @@ +import { Readability } from '@mozilla/readability'; +import * as cheerio from 'cheerio'; +import { convert } from 'html-to-text'; +import { JSDOM } from 'jsdom'; +import get from 'lodash/get'; +import set from 'lodash/set'; +import unset from 'lodash/unset'; +import * as mime from 'mime-types'; +import { getOAuth2AdditionalParameters } from 'n8n-nodes-base/dist/nodes/HttpRequest/GenericFunctions'; import type { - IExecuteFunctions, IDataObject, IHttpRequestOptions, IRequestOptionsSimplified, ExecutionError, NodeApiError, + ISupplyDataFunctions, } from 'n8n-workflow'; import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow'; - -import { getOAuth2AdditionalParameters } from 'n8n-nodes-base/dist/nodes/HttpRequest/GenericFunctions'; - -import set from 'lodash/set'; -import get from 'lodash/get'; -import unset from 'lodash/unset'; - -import cheerio from 'cheerio'; -import { convert } from 'html-to-text'; - -import { Readability } from '@mozilla/readability'; -import { JSDOM } from 'jsdom'; import { z } from 'zod'; -import type { DynamicZodObject } from '../../../types/zod.types'; + import type { ParameterInputType, ParametersValues, @@ -29,8 +26,9 @@ import type { SendIn, ToolParameter, } from './interfaces'; +import type { DynamicZodObject } from '../../../types/zod.types'; -const genericCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: number) => { +const genericCredentialRequest = async (ctx: ISupplyDataFunctions, itemIndex: number) => { const genericType = ctx.getNodeParameter('genericAuthType', itemIndex) as string; if (genericType === 'httpBasicAuth' || genericType === 'httpDigestAuth') { @@ -106,23 +104,22 @@ const genericCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: numbe }); }; -const predefinedCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: number) => { +const predefinedCredentialRequest = async (ctx: ISupplyDataFunctions, itemIndex: number) => { const predefinedType = ctx.getNodeParameter('nodeCredentialType', itemIndex) as string; const additionalOptions = getOAuth2AdditionalParameters(predefinedType); return async (options: IHttpRequestOptions) => { - return await ctx.helpers.requestWithAuthentication.call( + return await ctx.helpers.httpRequestWithAuthentication.call( ctx, predefinedType, options, additionalOptions && { oauth2: additionalOptions }, - itemIndex, ); }; }; export const configureHttpRequestFunction = async ( - ctx: IExecuteFunctions, + ctx: ISupplyDataFunctions, credentialsType: 'predefinedCredentialType' | 'genericCredentialType' | 'none', itemIndex: number, ) => { @@ -149,7 +146,7 @@ const defaultOptimizer = (response: T) => { return String(response); }; -const htmlOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: number) => { +const htmlOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number, maxLength: number) => { const cssSelector = ctx.getNodeParameter('cssSelector', itemIndex, '') as string; const onlyContent = ctx.getNodeParameter('onlyContent', itemIndex, false) as boolean; let elementsToOmit: string[] = []; @@ -176,6 +173,7 @@ const htmlOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: num ); } const returnData: string[] = []; + const html = cheerio.load(response); const htmlElements = html(cssSelector); @@ -216,7 +214,7 @@ const htmlOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: num }; }; -const textOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: number) => { +const textOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number, maxLength: number) => { return (response: string | IDataObject) => { if (typeof response === 'object') { try { @@ -247,7 +245,7 @@ const textOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: num }; }; -const jsonOptimizer = (ctx: IExecuteFunctions, itemIndex: number) => { +const jsonOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number) => { return (response: string): string => { let responseData: IDataObject | IDataObject[] | string = response; @@ -326,7 +324,7 @@ const jsonOptimizer = (ctx: IExecuteFunctions, itemIndex: number) => { }; }; -export const configureResponseOptimizer = (ctx: IExecuteFunctions, itemIndex: number) => { +export const configureResponseOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number) => { const optimizeResponse = ctx.getNodeParameter('optimizeResponse', itemIndex, false) as boolean; if (optimizeResponse) { @@ -471,7 +469,7 @@ const MODEL_INPUT_DESCRIPTION = { }; export const updateParametersAndOptions = (options: { - ctx: IExecuteFunctions; + ctx: ISupplyDataFunctions; itemIndex: number; toolParameters: ToolParameter[]; placeholdersDefinitions: PlaceholderDefinition[]; @@ -560,7 +558,7 @@ export const prepareToolDescription = ( }; export const configureToolFunction = ( - ctx: IExecuteFunctions, + ctx: ISupplyDataFunctions, itemIndex: number, toolParameters: ToolParameter[], requestOptions: IHttpRequestOptions, @@ -574,6 +572,7 @@ export const configureToolFunction = ( // Clone options and rawRequestOptions to avoid mutating the original objects const options: IHttpRequestOptions | null = structuredClone(requestOptions); const clonedRawRequestOptions: { [key: string]: string } = structuredClone(rawRequestOptions); + let fullResponse: any; let response: string = ''; let executionError: Error | undefined = undefined; @@ -732,8 +731,6 @@ export const configureToolFunction = ( } } } catch (error) { - console.error(error); - const errorMessage = 'Input provided by model is not valid'; if (error instanceof NodeOperationError) { @@ -749,11 +746,29 @@ export const configureToolFunction = ( if (options) { try { - response = optimizeResponse(await httpRequest(options)); + fullResponse = await httpRequest(options); } catch (error) { const httpCode = (error as NodeApiError).httpCode; response = `${httpCode ? `HTTP ${httpCode} ` : ''}There was an error: "${error.message}"`; } + + if (!response) { + try { + // Check if the response is binary data + if (fullResponse?.headers?.['content-type']) { + const contentType = fullResponse.headers['content-type'] as string; + const mimeType = contentType.split(';')[0].trim(); + + if (mime.charset(mimeType) !== 'UTF-8') { + throw new NodeOperationError(ctx.getNode(), 'Binary data is not supported'); + } + } + + response = optimizeResponse(fullResponse.body); + } catch (error) { + response = `There was an error: "${error.message}"`; + } + } } if (typeof response !== 'string') { diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolSerpApi/ToolSerpApi.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolSerpApi/ToolSerpApi.node.ts index c08553b96e..709b06b7ac 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolSerpApi/ToolSerpApi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolSerpApi/ToolSerpApi.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { SerpAPI } from '@langchain/community/tools/serpapi'; @@ -113,7 +113,7 @@ export class ToolSerpApi implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('serpApi'); const options = this.getNodeParameter('options', itemIndex) as object; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolVectorStore/ToolVectorStore.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolVectorStore/ToolVectorStore.node.ts index b4ea7c3321..b4016f06ca 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolVectorStore/ToolVectorStore.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolVectorStore/ToolVectorStore.node.ts @@ -1,4 +1,9 @@ -import type { IExecuteFunctions, INodeType, INodeTypeDescription, SupplyData } from 'n8n-workflow'; +import type { + INodeType, + INodeTypeDescription, + ISupplyDataFunctions, + SupplyData, +} from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; import { VectorStoreQATool } from 'langchain/tools'; @@ -82,7 +87,7 @@ export class ToolVectorStore implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const name = this.getNodeParameter('name', itemIndex) as string; const toolDescription = this.getNodeParameter('description', itemIndex) as string; const topK = this.getNodeParameter('topK', itemIndex, 4) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWikipedia/ToolWikipedia.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWikipedia/ToolWikipedia.node.ts index 0b3aeaff74..e462e38feb 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWikipedia/ToolWikipedia.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWikipedia/ToolWikipedia.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { WikipediaQueryRun } from '@langchain/community/tools/wikipedia_query_run'; @@ -43,7 +43,7 @@ export class ToolWikipedia implements INodeType { properties: [getConnectionHintNoticeField([NodeConnectionType.AiAgent])], }; - async supplyData(this: IExecuteFunctions): Promise { + async supplyData(this: ISupplyDataFunctions): Promise { const WikiTool = new WikipediaQueryRun(); WikiTool.description = diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWolframAlpha/ToolWolframAlpha.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWolframAlpha/ToolWolframAlpha.node.ts index 3a1f7ea2cc..93290e63ad 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWolframAlpha/ToolWolframAlpha.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWolframAlpha/ToolWolframAlpha.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import { WolframAlphaTool } from '@langchain/community/tools/wolframalpha'; @@ -49,7 +49,7 @@ export class ToolWolframAlpha implements INodeType { properties: [getConnectionHintNoticeField([NodeConnectionType.AiAgent])], }; - async supplyData(this: IExecuteFunctions): Promise { + async supplyData(this: ISupplyDataFunctions): Promise { const credentials = await this.getCredentials('wolframAlphaApi'); return { diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts index 352a727d11..f912e162d9 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts @@ -1,32 +1,33 @@ +import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager'; +import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; +import type { JSONSchema7 } from 'json-schema'; +import get from 'lodash/get'; +import isObject from 'lodash/isObject'; +import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces'; +import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode'; import type { - IExecuteFunctions, IExecuteWorkflowInfo, INodeExecutionData, INodeType, INodeTypeDescription, IWorkflowBase, + ISupplyDataFunctions, SupplyData, ExecutionError, IDataObject, INodeParameterResourceLocator, } from 'n8n-workflow'; import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow'; -import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces'; -import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode'; -import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; -import get from 'lodash/get'; -import isObject from 'lodash/isObject'; -import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager'; -import type { JSONSchema7 } from 'json-schema'; -import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; import type { DynamicZodObject } from '../../../types/zod.types'; -import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; import { jsonSchemaExampleField, schemaTypeField, inputSchemaField, } from '../../../utils/descriptions'; +import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing'; +import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; + export class ToolWorkflow implements INodeType { description: INodeTypeDescription = { displayName: 'Call n8n Workflow Tool', @@ -356,7 +357,7 @@ export class ToolWorkflow implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const name = this.getNodeParameter('name', itemIndex) as string; const description = this.getNodeParameter('description', itemIndex) as string; @@ -529,10 +530,9 @@ export class ToolWorkflow implements INodeType { ? generateSchema(jsonExample) : jsonParse(inputSchema); - const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); - const zodSchema = await zodSchemaSandbox.runCode(); + const zodSchema = convertJsonSchemaToZod(jsonSchema); - tool = new DynamicStructuredTool({ + tool = new DynamicStructuredTool({ schema: zodSchema, ...functionBase, }); diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts index fecf8d163a..cbd2cef75a 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts @@ -19,7 +19,7 @@ const insertFields: INodeProperties[] = [ }, ]; -export const VectorStoreInMemory = createVectorStoreNode({ +export class VectorStoreInMemory extends createVectorStoreNode({ meta: { displayName: 'In-Memory Vector Store', name: 'vectorStoreInMemory', @@ -56,4 +56,4 @@ export const VectorStoreInMemory = createVectorStoreNode({ void vectorStoreInstance.addDocuments(`${workflowId}__${memoryKey}`, documents, clearStore); }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemoryLoad/VectorStoreInMemoryLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemoryLoad/VectorStoreInMemoryLoad.node.ts index c85a245073..7bf48c3d8c 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemoryLoad/VectorStoreInMemoryLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemoryLoad/VectorStoreInMemoryLoad.node.ts @@ -1,10 +1,10 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { NodeConnectionType, - type SupplyData, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, + type SupplyData, } from 'n8n-workflow'; import type { Embeddings } from '@langchain/core/embeddings'; import { MemoryVectorStoreManager } from '../shared/MemoryVectorStoreManager'; @@ -59,7 +59,7 @@ export class VectorStoreInMemoryLoad implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const embeddings = (await this.getInputConnectionData( NodeConnectionType.AiEmbedding, itemIndex, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts index 0c9a148bec..8336958cc5 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts @@ -212,7 +212,7 @@ class ExtendedPGVectorStore extends PGVectorStore { } } -export const VectorStorePGVector = createVectorStoreNode({ +export class VectorStorePGVector extends createVectorStoreNode({ meta: { description: 'Work with your data in Postgresql with the PGVector extension', icon: 'file:postgres.svg', @@ -308,4 +308,4 @@ export const VectorStorePGVector = createVectorStoreNode({ await PGVectorStore.fromDocuments(documents, embeddings, config); }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index 85509a6287..d153979ef4 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -49,7 +49,7 @@ const insertFields: INodeProperties[] = [ }, ]; -export const VectorStorePinecone = createVectorStoreNode({ +export class VectorStorePinecone extends createVectorStoreNode({ meta: { displayName: 'Pinecone Vector Store', name: 'vectorStorePinecone', @@ -132,4 +132,4 @@ export const VectorStorePinecone = createVectorStoreNode({ pineconeIndex, }); }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts index 7cae9c9d85..d46bccd9f7 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts @@ -1,8 +1,8 @@ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { PineconeStoreParams } from '@langchain/pinecone'; @@ -84,7 +84,7 @@ export class VectorStorePineconeLoad implements INodeType { }, }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supplying data for Pinecone Load Vector Store'); const namespace = this.getNodeParameter('pineconeNamespace', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts index 5568243418..0b5859e0bc 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts @@ -78,7 +78,7 @@ const retrieveFields: INodeProperties[] = [ }, ]; -export const VectorStoreQdrant = createVectorStoreNode({ +export class VectorStoreQdrant extends createVectorStoreNode({ meta: { displayName: 'Qdrant Vector Store', name: 'vectorStoreQdrant', @@ -134,4 +134,4 @@ export const VectorStoreQdrant = createVectorStoreNode({ await QdrantVectorStore.fromDocuments(documents, embeddings, config); }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts index 0269a0cef2..549fcd5e7f 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts @@ -39,7 +39,7 @@ const retrieveFields: INodeProperties[] = [ const updateFields: INodeProperties[] = [...insertFields]; -export const VectorStoreSupabase = createVectorStoreNode({ +export class VectorStoreSupabase extends createVectorStoreNode({ meta: { description: 'Work with your data in Supabase Vector Store', icon: 'file:supabase.svg', @@ -109,4 +109,4 @@ export const VectorStoreSupabase = createVectorStoreNode({ } } }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts index bae6d0e1a9..f4bdc49e44 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts @@ -1,7 +1,7 @@ import { - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, NodeConnectionType, } from 'n8n-workflow'; @@ -81,7 +81,7 @@ export class VectorStoreSupabaseLoad implements INodeType { methods = { listSearch: { supabaseTableNameSearch } }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supply Supabase Load Vector Store'); const tableName = this.getNodeParameter('tableName', itemIndex, '', { diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts index d041b43d6b..184b720d31 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts @@ -44,7 +44,7 @@ const retrieveFields: INodeProperties[] = [ }, ]; -export const VectorStoreZep = createVectorStoreNode({ +export class VectorStoreZep extends createVectorStoreNode({ meta: { displayName: 'Zep Vector Store', name: 'vectorStoreZep', @@ -130,4 +130,4 @@ export const VectorStoreZep = createVectorStoreNode({ throw new NodeOperationError(context.getNode(), error as Error, { itemIndex }); } }, -}); +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts index 244c0a9843..dd30a0808e 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts @@ -1,8 +1,8 @@ import { NodeConnectionType, - type IExecuteFunctions, type INodeType, type INodeTypeDescription, + type ISupplyDataFunctions, type SupplyData, } from 'n8n-workflow'; import type { IZepConfig } from '@langchain/community/vectorstores/zep'; @@ -83,7 +83,7 @@ export class VectorStoreZepLoad implements INodeType { ], }; - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { this.logger.debug('Supplying data for Zep Load Vector Store'); const collectionName = this.getNodeParameter('collectionName', itemIndex) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryVectorStoreManager.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryVectorStoreManager.ts index 806a5129c5..1076fb93ba 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryVectorStoreManager.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryVectorStoreManager.ts @@ -7,11 +7,8 @@ export class MemoryVectorStoreManager { private vectorStoreBuffer: Map; - private embeddings: Embeddings; - - private constructor(embeddings: Embeddings) { + private constructor(private embeddings: Embeddings) { this.vectorStoreBuffer = new Map(); - this.embeddings = embeddings; } public static getInstance(embeddings: Embeddings): MemoryVectorStoreManager { diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index d487969073..2de9304fc5 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -1,28 +1,30 @@ /* eslint-disable n8n-nodes-base/node-filename-against-convention */ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ +import type { Document } from '@langchain/core/documents'; +import type { Embeddings } from '@langchain/core/embeddings'; import type { VectorStore } from '@langchain/core/vectorstores'; import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import type { + IExecuteFunctions, INodeCredentialDescription, INodeProperties, INodeExecutionData, - IExecuteFunctions, INodeTypeDescription, SupplyData, + ISupplyDataFunctions, INodeType, ILoadOptionsFunctions, INodeListSearchResult, Icon, INodePropertyOptions, } from 'n8n-workflow'; -import type { Embeddings } from '@langchain/core/embeddings'; -import type { Document } from '@langchain/core/documents'; -import { logWrapper } from '../../../utils/logWrapper'; -import { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; -import type { N8nBinaryLoader } from '../../../utils/N8nBinaryLoader'; -import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers'; -import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; + import { processDocument } from './processDocuments'; +import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers'; +import { logWrapper } from '../../../utils/logWrapper'; +import type { N8nBinaryLoader } from '../../../utils/N8nBinaryLoader'; +import { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; +import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update'; @@ -56,13 +58,13 @@ interface VectorStoreNodeConstructorArgs { retrieveFields?: INodeProperties[]; updateFields?: INodeProperties[]; populateVectorStore: ( - context: IExecuteFunctions, + context: ISupplyDataFunctions, embeddings: Embeddings, documents: Array>>, itemIndex: number, ) => Promise; getVectorStoreClient: ( - context: IExecuteFunctions, + context: ISupplyDataFunctions, filter: Record | undefined, embeddings: Embeddings, itemIndex: number, @@ -280,7 +282,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => }); resultData.push(...serializedDocs); - void logAiEvent(this, 'ai-vector-store-searched', { query: prompt }); + logAiEvent(this, 'ai-vector-store-searched', { query: prompt }); } return [resultData]; @@ -296,6 +298,9 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const resultData = []; for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + if (this.getExecutionCancelSignal()?.aborted) { + break; + } const itemData = items[itemIndex]; const { processedDocuments, serializedDocuments } = await processDocument( documentInput, @@ -307,7 +312,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => try { await args.populateVectorStore(this, embeddings, processedDocuments, itemIndex); - void logAiEvent(this, 'ai-vector-store-populated'); + logAiEvent(this, 'ai-vector-store-populated'); } catch (error) { throw error; } @@ -361,7 +366,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => ids: [documentId], }); - void logAiEvent(this, 'ai-vector-store-updated'); + logAiEvent(this, 'ai-vector-store-updated'); } catch (error) { throw error; } @@ -376,7 +381,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => ); } - async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve'; const filter = getMetadataFiltersValues(this, itemIndex); const embeddings = (await this.getInputConnectionData( diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index 4df337ed52..3672f73464 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/n8n-nodes-langchain", - "version": "1.63.0", + "version": "1.65.0", "description": "", "main": "index.js", "scripts": { @@ -124,6 +124,7 @@ "@types/cheerio": "^0.22.15", "@types/html-to-text": "^9.0.1", "@types/json-schema": "^7.0.15", + "@types/mime-types": "^2.1.0", "@types/pg": "^8.11.6", "@types/temp": "^0.9.1", "n8n-core": "workspace:*" @@ -167,10 +168,11 @@ "generate-schema": "2.6.0", "html-to-text": "9.0.5", "jsdom": "23.0.1", - "json-schema-to-zod": "2.1.0", + "@n8n/json-schema-to-zod": "workspace:*", "langchain": "0.3.2", "lodash": "catalog:", "mammoth": "1.7.2", + "mime-types": "2.1.35", "n8n-nodes-base": "workspace:*", "n8n-workflow": "workspace:*", "openai": "4.63.0", diff --git a/packages/@n8n/nodes-langchain/utils/N8nBinaryLoader.ts b/packages/@n8n/nodes-langchain/utils/N8nBinaryLoader.ts index 491bb03e28..53f4f95a74 100644 --- a/packages/@n8n/nodes-langchain/utils/N8nBinaryLoader.ts +++ b/packages/@n8n/nodes-langchain/utils/N8nBinaryLoader.ts @@ -1,6 +1,11 @@ import { pipeline } from 'stream/promises'; import { createWriteStream } from 'fs'; -import type { IBinaryData, IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; +import type { + IBinaryData, + IExecuteFunctions, + INodeExecutionData, + ISupplyDataFunctions, +} from 'n8n-workflow'; import { NodeOperationError, BINARY_ENCODING } from 'n8n-workflow'; import type { TextSplitter } from '@langchain/textsplitters'; @@ -26,25 +31,12 @@ const SUPPORTED_MIME_TYPES = { }; export class N8nBinaryLoader { - private context: IExecuteFunctions; - - private optionsPrefix: string; - - private binaryDataKey: string; - - private textSplitter?: TextSplitter; - constructor( - context: IExecuteFunctions, - optionsPrefix = '', - binaryDataKey = '', - textSplitter?: TextSplitter, - ) { - this.context = context; - this.textSplitter = textSplitter; - this.optionsPrefix = optionsPrefix; - this.binaryDataKey = binaryDataKey; - } + private context: IExecuteFunctions | ISupplyDataFunctions, + private optionsPrefix = '', + private binaryDataKey = '', + private textSplitter?: TextSplitter, + ) {} async processAll(items?: INodeExecutionData[]): Promise { const docs: Document[] = []; diff --git a/packages/@n8n/nodes-langchain/utils/N8nJsonLoader.ts b/packages/@n8n/nodes-langchain/utils/N8nJsonLoader.ts index 6cc4862d22..7c44d8a8f9 100644 --- a/packages/@n8n/nodes-langchain/utils/N8nJsonLoader.ts +++ b/packages/@n8n/nodes-langchain/utils/N8nJsonLoader.ts @@ -1,4 +1,9 @@ -import { type IExecuteFunctions, type INodeExecutionData, NodeOperationError } from 'n8n-workflow'; +import { + type IExecuteFunctions, + type INodeExecutionData, + type ISupplyDataFunctions, + NodeOperationError, +} from 'n8n-workflow'; import type { TextSplitter } from '@langchain/textsplitters'; import type { Document } from '@langchain/core/documents'; @@ -7,17 +12,11 @@ import { TextLoader } from 'langchain/document_loaders/fs/text'; import { getMetadataFiltersValues } from './helpers'; export class N8nJsonLoader { - private context: IExecuteFunctions; - - private optionsPrefix: string; - - private textSplitter?: TextSplitter; - - constructor(context: IExecuteFunctions, optionsPrefix = '', textSplitter?: TextSplitter) { - this.context = context; - this.textSplitter = textSplitter; - this.optionsPrefix = optionsPrefix; - } + constructor( + private context: IExecuteFunctions | ISupplyDataFunctions, + private optionsPrefix = '', + private textSplitter?: TextSplitter, + ) {} async processAll(items?: INodeExecutionData[]): Promise { const docs: Document[] = []; diff --git a/packages/@n8n/nodes-langchain/utils/N8nTool.ts b/packages/@n8n/nodes-langchain/utils/N8nTool.ts index bb8bab08bd..2cb89630f0 100644 --- a/packages/@n8n/nodes-langchain/utils/N8nTool.ts +++ b/packages/@n8n/nodes-langchain/utils/N8nTool.ts @@ -1,6 +1,6 @@ import type { DynamicStructuredToolInput } from '@langchain/core/tools'; import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; -import type { IExecuteFunctions, IDataObject } from 'n8n-workflow'; +import type { ISupplyDataFunctions, IDataObject } from 'n8n-workflow'; import { NodeConnectionType, jsonParse, NodeOperationError } from 'n8n-workflow'; import { StructuredOutputParser } from 'langchain/output_parsers'; import type { ZodTypeAny } from 'zod'; @@ -45,12 +45,11 @@ ALL parameters marked as required must be provided`; }; export class N8nTool extends DynamicStructuredTool { - private context: IExecuteFunctions; - - constructor(context: IExecuteFunctions, fields: DynamicStructuredToolInput) { + constructor( + private context: ISupplyDataFunctions, + fields: DynamicStructuredToolInput, + ) { super(fields); - - this.context = context; } asDynamicTool(): DynamicTool { diff --git a/packages/@n8n/nodes-langchain/utils/helpers.ts b/packages/@n8n/nodes-langchain/utils/helpers.ts index c70c8a8991..f1e02e9c9f 100644 --- a/packages/@n8n/nodes-langchain/utils/helpers.ts +++ b/packages/@n8n/nodes-langchain/utils/helpers.ts @@ -1,12 +1,18 @@ -import { NodeConnectionType, NodeOperationError, jsonStringify } from 'n8n-workflow'; -import type { AiEvent, IDataObject, IExecuteFunctions, IWebhookFunctions } from 'n8n-workflow'; +import type { BaseChatMessageHistory } from '@langchain/core/chat_history'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import type { BaseOutputParser } from '@langchain/core/output_parsers'; +import type { BaseLLM } from '@langchain/core/language_models/llms'; import type { BaseMessage } from '@langchain/core/messages'; import type { Tool } from '@langchain/core/tools'; -import type { BaseLLM } from '@langchain/core/language_models/llms'; import type { BaseChatMemory } from 'langchain/memory'; -import type { BaseChatMessageHistory } from '@langchain/core/chat_history'; +import { NodeConnectionType, NodeOperationError, jsonStringify } from 'n8n-workflow'; +import type { + AiEvent, + IDataObject, + IExecuteFunctions, + ISupplyDataFunctions, + IWebhookFunctions, +} from 'n8n-workflow'; + import { N8nTool } from './N8nTool'; function hasMethods(obj: unknown, ...methodNames: Array): obj is T { @@ -20,7 +26,7 @@ function hasMethods(obj: unknown, ...methodNames: Array): ob } export function getMetadataFiltersValues( - ctx: IExecuteFunctions, + ctx: IExecuteFunctions | ISupplyDataFunctions, itemIndex: number, ): Record | undefined { const options = ctx.getNodeParameter('options', itemIndex, {}); @@ -66,21 +72,6 @@ export function isToolsInstance(model: unknown): model is Tool { return namespace.includes('tools'); } -export async function getOptionalOutputParsers( - ctx: IExecuteFunctions, -): Promise>> { - let outputParsers: BaseOutputParser[] = []; - - if (ctx.getNodeParameter('hasOutputParser', 0, true) === true) { - outputParsers = (await ctx.getInputConnectionData( - NodeConnectionType.AiOutputParser, - 0, - )) as BaseOutputParser[]; - } - - return outputParsers; -} - export function getPromptInputByType(options: { ctx: IExecuteFunctions; i: number; @@ -108,7 +99,7 @@ export function getPromptInputByType(options: { } export function getSessionId( - ctx: IExecuteFunctions | IWebhookFunctions, + ctx: ISupplyDataFunctions | IWebhookFunctions, itemIndex: number, selectorKey = 'sessionIdType', autoSelect = 'fromInput', @@ -148,13 +139,13 @@ export function getSessionId( return sessionId; } -export async function logAiEvent( - executeFunctions: IExecuteFunctions, +export function logAiEvent( + executeFunctions: IExecuteFunctions | ISupplyDataFunctions, event: AiEvent, data?: IDataObject, ) { try { - await executeFunctions.logAiEvent(event, data ? jsonStringify(data) : undefined); + executeFunctions.logAiEvent(event, data ? jsonStringify(data) : undefined); } catch (error) { executeFunctions.logger.debug(`Error logging AI event: ${event}`); } diff --git a/packages/@n8n/nodes-langchain/utils/logWrapper.ts b/packages/@n8n/nodes-langchain/utils/logWrapper.ts index 8707726183..eca1431a4b 100644 --- a/packages/@n8n/nodes-langchain/utils/logWrapper.ts +++ b/packages/@n8n/nodes-langchain/utils/logWrapper.ts @@ -1,24 +1,21 @@ -import { NodeOperationError, NodeConnectionType } from 'n8n-workflow'; -import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; - -import type { Tool } from '@langchain/core/tools'; -import type { BaseMessage } from '@langchain/core/messages'; -import type { InputValues, MemoryVariables, OutputValues } from '@langchain/core/memory'; -import type { BaseChatMessageHistory } from '@langchain/core/chat_history'; -import type { BaseCallbackConfig, Callbacks } from '@langchain/core/callbacks/manager'; - -import { Embeddings } from '@langchain/core/embeddings'; -import { VectorStore } from '@langchain/core/vectorstores'; -import type { Document } from '@langchain/core/documents'; -import { TextSplitter } from '@langchain/textsplitters'; import type { BaseChatMemory } from '@langchain/community/memory/chat_memory'; +import type { BaseCallbackConfig, Callbacks } from '@langchain/core/callbacks/manager'; +import type { BaseChatMessageHistory } from '@langchain/core/chat_history'; +import type { Document } from '@langchain/core/documents'; +import { Embeddings } from '@langchain/core/embeddings'; +import type { InputValues, MemoryVariables, OutputValues } from '@langchain/core/memory'; +import type { BaseMessage } from '@langchain/core/messages'; import { BaseRetriever } from '@langchain/core/retrievers'; -import { BaseOutputParser, OutputParserException } from '@langchain/core/output_parsers'; -import { isObject } from 'lodash'; +import type { Tool } from '@langchain/core/tools'; +import { VectorStore } from '@langchain/core/vectorstores'; +import { TextSplitter } from '@langchain/textsplitters'; import type { BaseDocumentLoader } from 'langchain/dist/document_loaders/base'; -import { N8nJsonLoader } from './N8nJsonLoader'; -import { N8nBinaryLoader } from './N8nBinaryLoader'; +import type { IExecuteFunctions, INodeExecutionData, ISupplyDataFunctions } from 'n8n-workflow'; +import { NodeOperationError, NodeConnectionType } from 'n8n-workflow'; + import { logAiEvent, isToolsInstance, isBaseChatMemory, isBaseChatMessageHistory } from './helpers'; +import { N8nBinaryLoader } from './N8nBinaryLoader'; +import { N8nJsonLoader } from './N8nJsonLoader'; const errorsMap: { [key: string]: { message: string; description: string } } = { 'You exceeded your current quota, please check your plan and billing details.': { @@ -30,7 +27,7 @@ const errorsMap: { [key: string]: { message: string; description: string } } = { export async function callMethodAsync( this: T, parameters: { - executeFunctions: IExecuteFunctions; + executeFunctions: IExecuteFunctions | ISupplyDataFunctions; connectionType: NodeConnectionType; currentNodeRunIndex: number; method: (...args: any[]) => Promise; @@ -40,10 +37,6 @@ export async function callMethodAsync( try { return await parameters.method.call(this, ...parameters.arguments); } catch (e) { - // Langchain checks for OutputParserException to run retry chain - // for auto-fixing the output so skip wrapping in this case - if (e instanceof OutputParserException) throw e; - // Propagate errors from sub-nodes if (e.functionality === 'configuration-node') throw e; const connectedNode = parameters.executeFunctions.getNode(); @@ -63,7 +56,9 @@ export async function callMethodAsync( error, ); if (error.message) { - error.description = error.message; + if (!error.description) { + error.description = error.message; + } throw error; } throw new NodeOperationError( @@ -109,7 +104,6 @@ export function logWrapper( | Tool | BaseChatMemory | BaseChatMessageHistory - | BaseOutputParser | BaseRetriever | Embeddings | Document[] @@ -119,7 +113,7 @@ export function logWrapper( | VectorStore | N8nBinaryLoader | N8nJsonLoader, - executeFunctions: IExecuteFunctions, + executeFunctions: IExecuteFunctions | ISupplyDataFunctions, ) { return new Proxy(originalInstance, { get: (target, prop) => { @@ -196,7 +190,7 @@ export function logWrapper( const payload = { action: 'getMessages', response }; executeFunctions.addOutputData(connectionType, index, [[{ json: payload }]]); - void logAiEvent(executeFunctions, 'ai-messages-retrieved-from-memory', { response }); + logAiEvent(executeFunctions, 'ai-messages-retrieved-from-memory', { response }); return response; }; } else if (prop === 'addMessage' && 'addMessage' in target) { @@ -213,50 +207,12 @@ export function logWrapper( arguments: [message], }); - void logAiEvent(executeFunctions, 'ai-message-added-to-memory', { message }); + logAiEvent(executeFunctions, 'ai-message-added-to-memory', { message }); executeFunctions.addOutputData(connectionType, index, [[{ json: payload }]]); }; } } - // ========== BaseOutputParser ========== - if (originalInstance instanceof BaseOutputParser) { - if (prop === 'parse' && 'parse' in target) { - return async (text: string | Record): Promise => { - connectionType = NodeConnectionType.AiOutputParser; - const stringifiedText = isObject(text) ? JSON.stringify(text) : text; - const { index } = executeFunctions.addInputData(connectionType, [ - [{ json: { action: 'parse', text: stringifiedText } }], - ]); - - try { - const response = (await callMethodAsync.call(target, { - executeFunctions, - connectionType, - currentNodeRunIndex: index, - method: target[prop], - arguments: [stringifiedText], - })) as object; - - void logAiEvent(executeFunctions, 'ai-output-parsed', { text, response }); - executeFunctions.addOutputData(connectionType, index, [ - [{ json: { action: 'parse', response } }], - ]); - return response; - } catch (error) { - void logAiEvent(executeFunctions, 'ai-output-parsed', { - text, - response: error.message ?? error, - }); - executeFunctions.addOutputData(connectionType, index, [ - [{ json: { action: 'parse', response: error.message ?? error } }], - ]); - throw error; - } - }; - } - } - // ========== BaseRetriever ========== if (originalInstance instanceof BaseRetriever) { if (prop === 'getRelevantDocuments' && 'getRelevantDocuments' in target) { @@ -277,7 +233,7 @@ export function logWrapper( arguments: [query, config], })) as Array>>; - void logAiEvent(executeFunctions, 'ai-documents-retrieved', { query }); + logAiEvent(executeFunctions, 'ai-documents-retrieved', { query }); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; }; @@ -302,7 +258,7 @@ export function logWrapper( arguments: [documents], })) as number[][]; - void logAiEvent(executeFunctions, 'ai-document-embedded'); + logAiEvent(executeFunctions, 'ai-document-embedded'); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; }; @@ -322,7 +278,7 @@ export function logWrapper( method: target[prop], arguments: [query], })) as number[]; - void logAiEvent(executeFunctions, 'ai-query-embedded'); + logAiEvent(executeFunctions, 'ai-query-embedded'); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; }; @@ -367,7 +323,7 @@ export function logWrapper( arguments: [item, itemIndex], })) as number[]; - void logAiEvent(executeFunctions, 'ai-document-processed'); + logAiEvent(executeFunctions, 'ai-document-processed'); executeFunctions.addOutputData(connectionType, index, [ [{ json: { response }, pairedItem: { item: itemIndex } }], ]); @@ -393,7 +349,7 @@ export function logWrapper( arguments: [text], })) as string[]; - void logAiEvent(executeFunctions, 'ai-text-split'); + logAiEvent(executeFunctions, 'ai-text-split'); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; }; @@ -417,7 +373,7 @@ export function logWrapper( arguments: [query], })) as string; - void logAiEvent(executeFunctions, 'ai-tool-called', { query, response }); + logAiEvent(executeFunctions, 'ai-tool-called', { query, response }); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; }; @@ -447,7 +403,7 @@ export function logWrapper( arguments: [query, k, filter, _callbacks], })) as Array>>; - void logAiEvent(executeFunctions, 'ai-vector-store-searched', { query }); + logAiEvent(executeFunctions, 'ai-vector-store-searched', { query }); executeFunctions.addOutputData(connectionType, index, [[{ json: { response } }]]); return response; diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/ItemListOutputParser.ts b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nItemListOutputParser.ts similarity index 88% rename from packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/ItemListOutputParser.ts rename to packages/@n8n/nodes-langchain/utils/output_parsers/N8nItemListOutputParser.ts index 7e596a2b68..f24238690b 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserItemList/ItemListOutputParser.ts +++ b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nItemListOutputParser.ts @@ -1,9 +1,9 @@ import { BaseOutputParser, OutputParserException } from '@langchain/core/output_parsers'; -export class ItemListOutputParser extends BaseOutputParser { +export class N8nItemListOutputParser extends BaseOutputParser { lc_namespace = ['n8n-nodes-langchain', 'output_parsers', 'list_items']; - private numberOfItems: number | undefined; + private numberOfItems: number = 3; private separator: string; @@ -39,7 +39,7 @@ export class ItemListOutputParser extends BaseOutputParser { this.numberOfItems ? this.numberOfItems + ' ' : '' }items separated by`; - const numberOfExamples = this.numberOfItems ?? 3; + const numberOfExamples = this.numberOfItems; const examples: string[] = []; for (let i = 1; i <= numberOfExamples; i++) { @@ -48,4 +48,8 @@ export class ItemListOutputParser extends BaseOutputParser { return `${instructions} "${this.separator}" (for example: "${examples.join(this.separator)}")`; } + + getSchema() { + return; + } } diff --git a/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputFixingParser.ts b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputFixingParser.ts new file mode 100644 index 0000000000..eec3b0c187 --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputFixingParser.ts @@ -0,0 +1,86 @@ +import type { Callbacks } from '@langchain/core/callbacks/manager'; +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import type { AIMessage } from '@langchain/core/messages'; +import { BaseOutputParser } from '@langchain/core/output_parsers'; +import type { ISupplyDataFunctions } from 'n8n-workflow'; +import { NodeConnectionType } from 'n8n-workflow'; + +import type { N8nStructuredOutputParser } from './N8nStructuredOutputParser'; +import { NAIVE_FIX_PROMPT } from './prompt'; +import { logAiEvent } from '../helpers'; + +export class N8nOutputFixingParser extends BaseOutputParser { + lc_namespace = ['langchain', 'output_parsers', 'fix']; + + constructor( + private context: ISupplyDataFunctions, + private model: BaseLanguageModel, + private outputParser: N8nStructuredOutputParser, + ) { + super(); + } + + getRetryChain() { + return NAIVE_FIX_PROMPT.pipe(this.model); + } + + /** + * Attempts to parse the completion string using the output parser. + * If the initial parse fails, it tries to fix the output using a retry chain. + * @param completion The string to be parsed + * @returns The parsed response + * @throws Error if both parsing attempts fail + */ + async parse(completion: string, callbacks?: Callbacks) { + const { index } = this.context.addInputData(NodeConnectionType.AiOutputParser, [ + [{ json: { action: 'parse', text: completion } }], + ]); + + try { + // First attempt to parse the completion + const response = await this.outputParser.parse(completion, callbacks, (e) => e); + logAiEvent(this.context, 'ai-output-parsed', { text: completion, response }); + + this.context.addOutputData(NodeConnectionType.AiOutputParser, index, [ + [{ json: { action: 'parse', response } }], + ]); + + return response; + } catch (error) { + try { + // Second attempt: use retry chain to fix the output + const result = (await this.getRetryChain().invoke({ + completion, + error, + instructions: this.getFormatInstructions(), + })) as AIMessage; + + const resultText = result.content.toString(); + const parsed = await this.outputParser.parse(resultText, callbacks); + + // Add the successfully parsed output to the context + this.context.addOutputData(NodeConnectionType.AiOutputParser, index, [ + [{ json: { action: 'parse', response: parsed } }], + ]); + + return parsed; + } catch (autoParseError) { + // If both attempts fail, add the error to the output and throw + this.context.addOutputData(NodeConnectionType.AiOutputParser, index, autoParseError); + throw autoParseError; + } + } + } + + /** + * Method to get the format instructions for the parser. + * @returns The format instructions for the parser. + */ + getFormatInstructions() { + return this.outputParser.getFormatInstructions(); + } + + getSchema() { + return this.outputParser.schema; + } +} diff --git a/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputParser.ts b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputParser.ts new file mode 100644 index 0000000000..e9d23a0dea --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nOutputParser.ts @@ -0,0 +1,26 @@ +import type { IExecuteFunctions } from 'n8n-workflow'; +import { NodeConnectionType } from 'n8n-workflow'; + +import { N8nItemListOutputParser } from './N8nItemListOutputParser'; +import { N8nOutputFixingParser } from './N8nOutputFixingParser'; +import { N8nStructuredOutputParser } from './N8nStructuredOutputParser'; + +export type N8nOutputParser = + | N8nOutputFixingParser + | N8nStructuredOutputParser + | N8nItemListOutputParser; + +export { N8nOutputFixingParser, N8nItemListOutputParser, N8nStructuredOutputParser }; + +export async function getOptionalOutputParsers(ctx: IExecuteFunctions): Promise { + let outputParsers: N8nOutputParser[] = []; + + if (ctx.getNodeParameter('hasOutputParser', 0, true) === true) { + outputParsers = (await ctx.getInputConnectionData( + NodeConnectionType.AiOutputParser, + 0, + )) as N8nOutputParser[]; + } + + return outputParsers; +} diff --git a/packages/@n8n/nodes-langchain/utils/output_parsers/N8nStructuredOutputParser.ts b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nStructuredOutputParser.ts new file mode 100644 index 0000000000..a24052f5e1 --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/output_parsers/N8nStructuredOutputParser.ts @@ -0,0 +1,116 @@ +import type { Callbacks } from '@langchain/core/callbacks/manager'; +import { StructuredOutputParser } from 'langchain/output_parsers'; +import get from 'lodash/get'; +import type { ISupplyDataFunctions } from 'n8n-workflow'; +import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; +import { z } from 'zod'; + +import { logAiEvent } from '../helpers'; + +const STRUCTURED_OUTPUT_KEY = '__structured__output'; +const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object'; +const STRUCTURED_OUTPUT_ARRAY_KEY = '__structured__output__array'; + +export class N8nStructuredOutputParser extends StructuredOutputParser< + z.ZodType +> { + constructor( + private context: ISupplyDataFunctions, + zodSchema: z.ZodSchema, + ) { + super(zodSchema); + } + + lc_namespace = ['langchain', 'output_parsers', 'structured']; + + async parse( + text: string, + _callbacks?: Callbacks, + errorMapper?: (error: Error) => Error, + ): Promise { + const { index } = this.context.addInputData(NodeConnectionType.AiOutputParser, [ + [{ json: { action: 'parse', text } }], + ]); + try { + const parsed = await super.parse(text); + + const result = (get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_OBJECT_KEY]) ?? + get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_ARRAY_KEY]) ?? + get(parsed, STRUCTURED_OUTPUT_KEY) ?? + parsed) as Record; + + logAiEvent(this.context, 'ai-output-parsed', { text, response: result }); + + this.context.addOutputData(NodeConnectionType.AiOutputParser, index, [ + [{ json: { action: 'parse', response: result } }], + ]); + + return result; + } catch (e) { + const nodeError = new NodeOperationError( + this.context.getNode(), + "Model output doesn't fit required format", + { + description: + "To continue the execution when this happens, change the 'On Error' parameter in the root node's settings", + }, + ); + + logAiEvent(this.context, 'ai-output-parsed', { + text, + response: e.message ?? e, + }); + + this.context.addOutputData(NodeConnectionType.AiOutputParser, index, nodeError); + if (errorMapper) { + throw errorMapper(e); + } + + throw nodeError; + } + } + + static async fromZodJsonSchema( + zodSchema: z.ZodSchema, + nodeVersion: number, + context: ISupplyDataFunctions, + ): Promise { + let returnSchema: z.ZodType; + if (nodeVersion === 1) { + returnSchema = z.object({ + [STRUCTURED_OUTPUT_KEY]: z + .object({ + [STRUCTURED_OUTPUT_OBJECT_KEY]: zodSchema.optional(), + [STRUCTURED_OUTPUT_ARRAY_KEY]: z.array(zodSchema).optional(), + }) + .describe( + `Wrapper around the output data. It can only contain ${STRUCTURED_OUTPUT_OBJECT_KEY} or ${STRUCTURED_OUTPUT_ARRAY_KEY} but never both.`, + ) + .refine( + (data) => { + // Validate that one and only one of the properties exists + return ( + Boolean(data[STRUCTURED_OUTPUT_OBJECT_KEY]) !== + Boolean(data[STRUCTURED_OUTPUT_ARRAY_KEY]) + ); + }, + { + message: + 'One and only one of __structured__output__object and __structured__output__array should be present.', + path: [STRUCTURED_OUTPUT_KEY], + }, + ), + }); + } else { + returnSchema = z.object({ + output: zodSchema.optional(), + }); + } + + return new N8nStructuredOutputParser(context, returnSchema); + } + + getSchema() { + return this.schema; + } +} diff --git a/packages/@n8n/nodes-langchain/utils/output_parsers/prompt.ts b/packages/@n8n/nodes-langchain/utils/output_parsers/prompt.ts new file mode 100644 index 0000000000..47599d230c --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/output_parsers/prompt.ts @@ -0,0 +1,20 @@ +import { PromptTemplate } from '@langchain/core/prompts'; + +export const NAIVE_FIX_TEMPLATE = `Instructions: +-------------- +{instructions} +-------------- +Completion: +-------------- +{completion} +-------------- + +Above, the Completion did not satisfy the constraints given in the Instructions. +Error: +-------------- +{error} +-------------- + +Please try again. Please only respond with an answer that satisfies the constraints laid out in the Instructions:`; + +export const NAIVE_FIX_PROMPT = PromptTemplate.fromTemplate(NAIVE_FIX_TEMPLATE); diff --git a/packages/@n8n/nodes-langchain/utils/schemaParsing.ts b/packages/@n8n/nodes-langchain/utils/schemaParsing.ts index 0591483e2c..592f1597c2 100644 --- a/packages/@n8n/nodes-langchain/utils/schemaParsing.ts +++ b/packages/@n8n/nodes-langchain/utils/schemaParsing.ts @@ -1,67 +1,10 @@ -import { makeResolverFromLegacyOptions } from '@n8n/vm2'; +import { jsonSchemaToZod } from '@n8n/json-schema-to-zod'; import { json as generateJsonSchema } from 'generate-schema'; import type { SchemaObject } from 'generate-schema'; import type { JSONSchema7 } from 'json-schema'; -import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox'; -import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; import type { IExecuteFunctions } from 'n8n-workflow'; import { NodeOperationError, jsonParse } from 'n8n-workflow'; - -const vmResolver = makeResolverFromLegacyOptions({ - external: { - modules: ['json-schema-to-zod', 'zod'], - transitive: false, - }, - resolve(moduleName, parentDirname) { - if (moduleName === 'json-schema-to-zod') { - return require.resolve( - '@n8n/n8n-nodes-langchain/node_modules/json-schema-to-zod/dist/cjs/jsonSchemaToZod.js', - { - paths: [parentDirname], - }, - ); - } - if (moduleName === 'zod') { - return require.resolve('@n8n/n8n-nodes-langchain/node_modules/zod.cjs', { - paths: [parentDirname], - }); - } - return; - }, - builtin: [], -}); - -export function getSandboxWithZod(ctx: IExecuteFunctions, schema: JSONSchema7, itemIndex: number) { - const context = getSandboxContext.call(ctx, itemIndex); - let itemSchema: JSONSchema7 = schema; - try { - // If the root type is not defined, we assume it's an object - if (itemSchema.type === undefined) { - itemSchema = { - type: 'object', - properties: itemSchema.properties ?? (itemSchema as { [key: string]: JSONSchema7 }), - }; - } - } catch (error) { - throw new NodeOperationError(ctx.getNode(), 'Error during parsing of JSON Schema.'); - } - - // Make sure to remove the description from root schema - const { description, ...restOfSchema } = itemSchema; - const sandboxedSchema = new JavaScriptSandbox( - context, - ` - const { z } = require('zod'); - const { parseSchema } = require('json-schema-to-zod'); - const zodSchema = parseSchema(${JSON.stringify(restOfSchema)}); - const itemSchema = new Function('z', 'return (' + zodSchema + ')')(z) - return itemSchema - `, - ctx.helpers, - { resolver: vmResolver }, - ); - return sandboxedSchema; -} +import type { z } from 'zod'; export function generateSchema(schemaString: string): JSONSchema7 { const parsedSchema = jsonParse(schemaString); @@ -69,6 +12,10 @@ export function generateSchema(schemaString: string): JSONSchema7 { return generateJsonSchema(parsedSchema) as JSONSchema7; } +export function convertJsonSchemaToZod(schema: JSONSchema7) { + return jsonSchemaToZod(schema); +} + export function throwIfToolSchema(ctx: IExecuteFunctions, error: Error) { if (error?.message?.includes('tool input did not match expected schema')) { throw new NodeOperationError( diff --git a/packages/@n8n/permissions/package.json b/packages/@n8n/permissions/package.json index 0dbc5ee515..d92c2c20ac 100644 --- a/packages/@n8n/permissions/package.json +++ b/packages/@n8n/permissions/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/permissions", - "version": "0.14.0", + "version": "0.15.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/permissions/src/constants.ts b/packages/@n8n/permissions/src/constants.ts index c43677e843..7a0ebf2cb1 100644 --- a/packages/@n8n/permissions/src/constants.ts +++ b/packages/@n8n/permissions/src/constants.ts @@ -3,6 +3,7 @@ export const RESOURCES = { annotationTag: [...DEFAULT_OPERATIONS] as const, auditLogs: ['manage'] as const, banner: ['dismiss'] as const, + community: ['register'] as const, communityPackage: ['install', 'uninstall', 'update', 'list', 'manage'] as const, credential: ['share', 'move', ...DEFAULT_OPERATIONS] as const, externalSecretsProvider: ['sync', ...DEFAULT_OPERATIONS] as const, diff --git a/packages/@n8n/permissions/src/types.ts b/packages/@n8n/permissions/src/types.ts index 1a78f79f15..b36fb792ae 100644 --- a/packages/@n8n/permissions/src/types.ts +++ b/packages/@n8n/permissions/src/types.ts @@ -1,71 +1,22 @@ -import type { DEFAULT_OPERATIONS, RESOURCES } from './constants'; +import type { RESOURCES } from './constants'; -export type DefaultOperations = (typeof DEFAULT_OPERATIONS)[number]; export type Resource = keyof typeof RESOURCES; export type ResourceScope< R extends Resource, - Operation extends string = DefaultOperations, + Operation extends (typeof RESOURCES)[R][number] = (typeof RESOURCES)[R][number], > = `${R}:${Operation}`; export type WildcardScope = `${Resource}:*` | '*'; -export type AnnotationTagScope = ResourceScope<'annotationTag'>; -export type AuditLogsScope = ResourceScope<'auditLogs', 'manage'>; -export type BannerScope = ResourceScope<'banner', 'dismiss'>; -export type CommunityPackageScope = ResourceScope< - 'communityPackage', - 'install' | 'uninstall' | 'update' | 'list' | 'manage' ->; -export type CredentialScope = ResourceScope<'credential', DefaultOperations | 'share' | 'move'>; -export type ExternalSecretScope = ResourceScope<'externalSecret', 'list' | 'use'>; -export type ExternalSecretProviderScope = ResourceScope< - 'externalSecretsProvider', - DefaultOperations | 'sync' ->; -export type EventBusDestinationScope = ResourceScope< - 'eventBusDestination', - DefaultOperations | 'test' ->; -export type LdapScope = ResourceScope<'ldap', 'manage' | 'sync'>; -export type LicenseScope = ResourceScope<'license', 'manage'>; -export type LogStreamingScope = ResourceScope<'logStreaming', 'manage'>; -export type OrchestrationScope = ResourceScope<'orchestration', 'read' | 'list'>; -export type ProjectScope = ResourceScope<'project'>; -export type SamlScope = ResourceScope<'saml', 'manage'>; -export type SecurityAuditScope = ResourceScope<'securityAudit', 'generate'>; -export type SourceControlScope = ResourceScope<'sourceControl', 'pull' | 'push' | 'manage'>; -export type TagScope = ResourceScope<'tag'>; -export type UserScope = ResourceScope<'user', DefaultOperations | 'resetPassword' | 'changeRole'>; -export type VariableScope = ResourceScope<'variable'>; -export type WorkersViewScope = ResourceScope<'workersView', 'manage'>; -export type WorkflowScope = ResourceScope< - 'workflow', - DefaultOperations | 'share' | 'execute' | 'move' ->; +// This is purely an intermediary type. +// If we tried to do use `ResourceScope` directly we'd end +// up with all resources having all scopes (e.g. `ldap:uninstall`). +type AllScopesObject = { + [R in Resource]: ResourceScope; +}; -export type Scope = - | AnnotationTagScope - | AuditLogsScope - | BannerScope - | CommunityPackageScope - | CredentialScope - | ExternalSecretProviderScope - | ExternalSecretScope - | EventBusDestinationScope - | LdapScope - | LicenseScope - | LogStreamingScope - | OrchestrationScope - | ProjectScope - | SamlScope - | SecurityAuditScope - | SourceControlScope - | TagScope - | UserScope - | VariableScope - | WorkersViewScope - | WorkflowScope; +export type Scope = AllScopesObject[K]; export type ScopeLevel = 'global' | 'project' | 'resource'; export type GetScopeLevel = Record; diff --git a/packages/@n8n/task-runner/package.json b/packages/@n8n/task-runner/package.json index a82b97975d..fca6b9a22c 100644 --- a/packages/@n8n/task-runner/package.json +++ b/packages/@n8n/task-runner/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/task-runner", - "version": "1.1.0", + "version": "1.3.0", "scripts": { "clean": "rimraf dist .turbo", "start": "node dist/start.js", @@ -13,7 +13,7 @@ "test:watch": "jest --watch", "lint": "eslint . --quiet", "lintfix": "eslint . --fix", - "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" + "watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\"" }, "main": "dist/start.js", "module": "src/start.ts", @@ -22,9 +22,11 @@ "dist/**/*" ], "dependencies": { + "@n8n/config": "workspace:*", "n8n-workflow": "workspace:*", "n8n-core": "workspace:*", "nanoid": "^3.3.6", + "typedi": "catalog:", "ws": "^8.18.0" }, "devDependencies": { diff --git a/packages/@n8n/task-runner/src/__tests__/node-types.test.ts b/packages/@n8n/task-runner/src/__tests__/node-types.test.ts new file mode 100644 index 0000000000..c102c80df3 --- /dev/null +++ b/packages/@n8n/task-runner/src/__tests__/node-types.test.ts @@ -0,0 +1,66 @@ +import type { INodeTypeDescription } from 'n8n-workflow'; + +import { TaskRunnerNodeTypes } from '../node-types'; + +const SINGLE_VERSIONED = { name: 'single-versioned', version: 1 }; + +const SINGLE_UNVERSIONED = { name: 'single-unversioned' }; + +const MULTI_VERSIONED = { name: 'multi-versioned', version: [1, 2] }; + +const SPLIT_VERSIONED = [ + { name: 'split-versioned', version: 1 }, + { name: 'split-versioned', version: 2 }, +]; + +const TYPES: INodeTypeDescription[] = [ + SINGLE_VERSIONED, + SINGLE_UNVERSIONED, + MULTI_VERSIONED, + ...SPLIT_VERSIONED, +] as INodeTypeDescription[]; + +describe('TaskRunnerNodeTypes', () => { + describe('getByNameAndVersion', () => { + let nodeTypes: TaskRunnerNodeTypes; + + beforeEach(() => { + nodeTypes = new TaskRunnerNodeTypes(TYPES); + }); + + it('should return undefined if not found', () => { + expect(nodeTypes.getByNameAndVersion('unknown', 1)).toBeUndefined(); + }); + + it('should return highest versioned node type if no version is given', () => { + expect(nodeTypes.getByNameAndVersion('split-versioned')).toEqual({ + description: SPLIT_VERSIONED[1], + }); + }); + + it('should return specified version for split version', () => { + expect(nodeTypes.getByNameAndVersion('split-versioned', 1)).toEqual({ + description: SPLIT_VERSIONED[0], + }); + }); + + it('should return undefined on unknown version', () => { + expect(nodeTypes.getByNameAndVersion('split-versioned', 3)).toBeUndefined(); + }); + + it('should return specified version for multi version', () => { + expect(nodeTypes.getByNameAndVersion('multi-versioned', 1)).toEqual({ + description: MULTI_VERSIONED, + }); + expect(nodeTypes.getByNameAndVersion('multi-versioned', 2)).toEqual({ + description: MULTI_VERSIONED, + }); + }); + + it('should default to DEFAULT_NODETYPE_VERSION if no version specified', () => { + expect(nodeTypes.getByNameAndVersion('single-unversioned', 1)).toEqual({ + description: SINGLE_UNVERSIONED, + }); + }); + }); +}); diff --git a/packages/@n8n/task-runner/src/authenticator.ts b/packages/@n8n/task-runner/src/authenticator.ts deleted file mode 100644 index 7edb4cadf6..0000000000 --- a/packages/@n8n/task-runner/src/authenticator.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { ApplicationError } from 'n8n-workflow'; -import * as a from 'node:assert/strict'; - -export type AuthOpts = { - n8nUri: string; - authToken: string; -}; - -/** - * Requests a one-time token that can be used to establish a task runner connection - */ -export async function authenticate(opts: AuthOpts) { - try { - const authEndpoint = `http://${opts.n8nUri}/runners/auth`; - const response = await fetch(authEndpoint, { - method: 'POST', - headers: { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - token: opts.authToken, - }), - }); - - if (!response.ok) { - throw new ApplicationError( - `Invalid response status ${response.status}: ${await response.text()}`, - ); - } - - const { data } = (await response.json()) as { data: { token: string } }; - const grantToken = data.token; - a.ok(grantToken); - - return grantToken; - } catch (e) { - console.error(e); - const error = e as Error; - throw new ApplicationError( - `Could not connect to n8n message broker ${opts.n8nUri}: ${error.message}`, - { - cause: error, - }, - ); - } -} diff --git a/packages/@n8n/task-runner/src/config/base-runner-config.ts b/packages/@n8n/task-runner/src/config/base-runner-config.ts new file mode 100644 index 0000000000..01e00c177a --- /dev/null +++ b/packages/@n8n/task-runner/src/config/base-runner-config.ts @@ -0,0 +1,16 @@ +import { Config, Env } from '@n8n/config'; + +@Config +export class BaseRunnerConfig { + @Env('N8N_RUNNERS_N8N_URI') + n8nUri: string = '127.0.0.1:5679'; + + @Env('N8N_RUNNERS_GRANT_TOKEN') + grantToken: string = ''; + + @Env('N8N_RUNNERS_MAX_PAYLOAD') + maxPayloadSize: number = 1024 * 1024 * 1024; + + @Env('N8N_RUNNERS_MAX_CONCURRENCY') + maxConcurrency: number = 5; +} diff --git a/packages/@n8n/task-runner/src/config/js-runner-config.ts b/packages/@n8n/task-runner/src/config/js-runner-config.ts new file mode 100644 index 0000000000..4cba6f1d98 --- /dev/null +++ b/packages/@n8n/task-runner/src/config/js-runner-config.ts @@ -0,0 +1,10 @@ +import { Config, Env } from '@n8n/config'; + +@Config +export class JsRunnerConfig { + @Env('NODE_FUNCTION_ALLOW_BUILTIN') + allowedBuiltInModules: string = ''; + + @Env('NODE_FUNCTION_ALLOW_EXTERNAL') + allowedExternalModules: string = ''; +} diff --git a/packages/@n8n/task-runner/src/config/main-config.ts b/packages/@n8n/task-runner/src/config/main-config.ts new file mode 100644 index 0000000000..a290c0c380 --- /dev/null +++ b/packages/@n8n/task-runner/src/config/main-config.ts @@ -0,0 +1,13 @@ +import { Config, Nested } from '@n8n/config'; + +import { BaseRunnerConfig } from './base-runner-config'; +import { JsRunnerConfig } from './js-runner-config'; + +@Config +export class MainConfig { + @Nested + baseRunnerConfig!: BaseRunnerConfig; + + @Nested + jsRunnerConfig!: JsRunnerConfig; +} diff --git a/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts b/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts index 499105f39d..36f3c5afa2 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts @@ -4,7 +4,6 @@ import fs from 'node:fs'; import { builtinModules } from 'node:module'; import { ValidationError } from '@/js-task-runner/errors/validation-error'; -import type { JsTaskRunnerOpts } from '@/js-task-runner/js-task-runner'; import { JsTaskRunner, type AllCodeTaskData, @@ -13,17 +12,27 @@ import { import type { Task } from '@/task-runner'; import { newAllCodeTaskData, newTaskWithSettings, withPairedItem, wrapIntoJson } from './test-data'; +import type { JsRunnerConfig } from '../../config/js-runner-config'; +import { MainConfig } from '../../config/main-config'; import { ExecutionError } from '../errors/execution-error'; jest.mock('ws'); +const defaultConfig = new MainConfig(); + describe('JsTaskRunner', () => { - const createRunnerWithOpts = (opts: Partial = {}) => + const createRunnerWithOpts = (opts: Partial = {}) => new JsTaskRunner({ - wsUrl: 'ws://localhost', - grantToken: 'grantToken', - maxConcurrency: 1, - ...opts, + baseRunnerConfig: { + ...defaultConfig.baseRunnerConfig, + grantToken: 'grantToken', + maxConcurrency: 1, + n8nUri: 'localhost', + }, + jsRunnerConfig: { + ...defaultConfig.jsRunnerConfig, + ...opts, + }, }); const defaultTaskRunner = createRunnerWithOpts(); @@ -189,6 +198,25 @@ describe('JsTaskRunner', () => { ['{ wf: $workflow }', { wf: { active: true, id: '1', name: 'Test Workflow' } }], ['$vars', { var: 'value' }], ], + 'Node.js internal functions': [ + ['typeof Function', 'function'], + ['typeof eval', 'function'], + ['typeof setTimeout', 'function'], + ['typeof setInterval', 'function'], + ['typeof setImmediate', 'function'], + ['typeof clearTimeout', 'function'], + ['typeof clearInterval', 'function'], + ['typeof clearImmediate', 'function'], + ], + 'JS built-ins': [ + ['typeof btoa', 'function'], + ['typeof atob', 'function'], + ['typeof TextDecoder', 'function'], + ['typeof TextDecoderStream', 'function'], + ['typeof TextEncoder', 'function'], + ['typeof TextEncoderStream', 'function'], + ['typeof FormData', 'function'], + ], }; for (const [groupName, tests] of Object.entries(testGroups)) { @@ -281,6 +309,34 @@ describe('JsTaskRunner', () => { expect(outcome.result).toEqual([wrapIntoJson({ val: undefined })]); }); }); + + it('should allow access to Node.js Buffers', async () => { + const outcomeAll = await execTaskWithParams({ + task: newTaskWithSettings({ + code: 'return { val: Buffer.from("test-buffer").toString() }', + nodeMode: 'runOnceForAllItems', + }), + taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), { + envProviderState: undefined, + }), + }); + + expect(outcomeAll.result).toEqual([wrapIntoJson({ val: 'test-buffer' })]); + + const outcomePer = await execTaskWithParams({ + task: newTaskWithSettings({ + code: 'return { val: Buffer.from("test-buffer").toString() }', + nodeMode: 'runOnceForEachItem', + }), + taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), { + envProviderState: undefined, + }), + }); + + expect(outcomePer.result).toEqual([ + { ...wrapIntoJson({ val: 'test-buffer' }), pairedItem: { item: 0 } }, + ]); + }); }); describe('runOnceForAllItems', () => { @@ -744,19 +800,20 @@ describe('JsTaskRunner', () => { await runner.receivedSettings(taskId, task.settings); - expect(sendSpy).toHaveBeenCalledWith( - JSON.stringify({ - type: 'runner:taskerror', - taskId, - error: { - message: 'unknown is not defined [line 1]', - description: 'ReferenceError', - lineNumber: 1, - }, - }), - ); - - console.log('DONE'); - }, 1000); + expect(sendSpy).toHaveBeenCalled(); + const calledWith = sendSpy.mock.calls[0][0] as string; + expect(typeof calledWith).toBe('string'); + const calledObject = JSON.parse(calledWith); + expect(calledObject).toEqual({ + type: 'runner:taskerror', + taskId, + error: { + stack: expect.any(String), + message: 'unknown is not defined [line 1]', + description: 'ReferenceError', + lineNumber: 1, + }, + }); + }); }); }); diff --git a/packages/@n8n/task-runner/src/js-task-runner/errors/__tests__/execution-error.test.ts b/packages/@n8n/task-runner/src/js-task-runner/errors/__tests__/execution-error.test.ts index 3777940021..c85e52f977 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/errors/__tests__/execution-error.test.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/errors/__tests__/execution-error.test.ts @@ -42,6 +42,7 @@ describe('ExecutionError', () => { expect(JSON.stringify(executionError)).toBe( JSON.stringify({ + stack: defaultStack, message: 'a.unknown is not a function [line 2, for item 1]', description: 'TypeError', itemIndex: 1, diff --git a/packages/@n8n/task-runner/src/js-task-runner/errors/serializable-error.ts b/packages/@n8n/task-runner/src/js-task-runner/errors/serializable-error.ts index cd0e568de0..ea6321746b 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/errors/serializable-error.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/errors/serializable-error.ts @@ -1,3 +1,24 @@ +/** + * Makes the given error's `message` and `stack` properties enumerable + * so they can be serialized with JSON.stringify + */ +export function makeSerializable(error: Error) { + Object.defineProperties(error, { + message: { + value: error.message, + enumerable: true, + configurable: true, + }, + stack: { + value: error.stack, + enumerable: true, + configurable: true, + }, + }); + + return error; +} + /** * Error that has its message property serialized as well. Used to transport * errors over the wire. @@ -6,16 +27,6 @@ export abstract class SerializableError extends Error { constructor(message: string) { super(message); - // So it is serialized as well - this.makeMessageEnumerable(); - } - - private makeMessageEnumerable() { - Object.defineProperty(this, 'message', { - value: this.message, - enumerable: true, // This makes the message property enumerable - writable: true, - configurable: true, - }); + makeSerializable(this); } } diff --git a/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts b/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts index 5bf2e06f26..40ee12af2c 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts @@ -7,7 +7,6 @@ import { import type { CodeExecutionMode, INode, - INodeType, ITaskDataConnections, IWorkflowExecuteAdditionalData, WorkflowParameters, @@ -27,9 +26,11 @@ import { type Task, TaskRunner } from '@/task-runner'; import { isErrorLike } from './errors/error-like'; import { ExecutionError } from './errors/execution-error'; +import { makeSerializable } from './errors/serializable-error'; import type { RequireResolver } from './require-resolver'; import { createRequireResolver } from './require-resolver'; import { validateRunForAllItemsOutput, validateRunForEachItemOutput } from './result-validation'; +import type { MainConfig } from '../config/main-config'; export interface JSExecSettings { code: string; @@ -76,23 +77,6 @@ export interface AllCodeTaskData { additionalData: PartialAdditionalData; } -export interface JsTaskRunnerOpts { - wsUrl: string; - grantToken: string; - maxConcurrency: number; - name?: string; - /** - * List of built-in nodejs modules that are allowed to be required in the - * execution sandbox. Asterisk (*) can be used to allow all. - */ - allowedBuiltInModules?: string; - /** - * List of npm modules that are allowed to be required in the execution - * sandbox. Asterisk (*) can be used to allow all. - */ - allowedExternalModules?: string; -} - type CustomConsole = { log: (...args: unknown[]) => void; }; @@ -100,22 +84,20 @@ type CustomConsole = { export class JsTaskRunner extends TaskRunner { private readonly requireResolver: RequireResolver; - constructor({ - grantToken, - maxConcurrency, - wsUrl, - name = 'JS Task Runner', - allowedBuiltInModules, - allowedExternalModules, - }: JsTaskRunnerOpts) { - super('javascript', wsUrl, grantToken, maxConcurrency, name); + constructor(config: MainConfig, name = 'JS Task Runner') { + super({ + taskType: 'javascript', + name, + ...config.baseRunnerConfig, + }); + const { jsRunnerConfig } = config; const parseModuleAllowList = (moduleList: string) => moduleList === '*' ? null : new Set(moduleList.split(',').map((x) => x.trim())); this.requireResolver = createRequireResolver({ - allowedBuiltInModules: parseModuleAllowList(allowedBuiltInModules ?? ''), - allowedExternalModules: parseModuleAllowList(allowedExternalModules ?? ''), + allowedBuiltInModules: parseModuleAllowList(jsRunnerConfig.allowedBuiltInModules ?? ''), + allowedExternalModules: parseModuleAllowList(jsRunnerConfig.allowedExternalModules ?? ''), }); } @@ -128,17 +110,7 @@ export class JsTaskRunner extends TaskRunner { const workflowParams = allData.workflow; const workflow = new Workflow({ ...workflowParams, - nodeTypes: { - getByNameAndVersion() { - return undefined as unknown as INodeType; - }, - getByName() { - return undefined as unknown as INodeType; - }, - getKnownTypes() { - return {}; - }, - }, + nodeTypes: this.nodeTypes, }); const customConsole = { @@ -163,6 +135,30 @@ export class JsTaskRunner extends TaskRunner { }; } + private getNativeVariables() { + return { + // Exposed Node.js globals in vm2 + Buffer, + Function, + eval, + setTimeout, + setInterval, + setImmediate, + clearTimeout, + clearInterval, + clearImmediate, + + // Missing JS natives + btoa, + atob, + TextDecoder, + TextDecoderStream, + TextEncoder, + TextEncoderStream, + FormData, + }; + } + /** * Executes the requested code for all items in a single run */ @@ -180,15 +176,16 @@ export class JsTaskRunner extends TaskRunner { require: this.requireResolver, module: {}, console: customConsole, - items: inputItems, + + ...this.getNativeVariables(), ...dataProxy, ...this.buildRpcCallObject(taskId), }; try { const result = (await runInNewContext( - `module.exports = async function VmCodeWrapper() {${settings.code}\n}()`, + `globalThis.global = globalThis; module.exports = async function VmCodeWrapper() {${settings.code}\n}()`, context, )) as TaskResultData['result']; @@ -231,6 +228,7 @@ export class JsTaskRunner extends TaskRunner { console: customConsole, item, + ...this.getNativeVariables(), ...dataProxy, ...this.buildRpcCallObject(taskId), }; @@ -312,7 +310,7 @@ export class JsTaskRunner extends TaskRunner { private toExecutionErrorIfNeeded(error: unknown): Error { if (error instanceof Error) { - return error; + return makeSerializable(error); } if (isErrorLike(error)) { diff --git a/packages/@n8n/task-runner/src/node-types.ts b/packages/@n8n/task-runner/src/node-types.ts new file mode 100644 index 0000000000..046321bff9 --- /dev/null +++ b/packages/@n8n/task-runner/src/node-types.ts @@ -0,0 +1,64 @@ +import { + ApplicationError, + type IDataObject, + type INodeType, + type INodeTypeDescription, + type INodeTypes, + type IVersionedNodeType, +} from 'n8n-workflow'; + +type VersionedTypes = Map; + +export const DEFAULT_NODETYPE_VERSION = 1; + +export class TaskRunnerNodeTypes implements INodeTypes { + private nodeTypesByVersion: Map; + + constructor(nodeTypes: INodeTypeDescription[]) { + this.nodeTypesByVersion = this.parseNodeTypes(nodeTypes); + } + + private parseNodeTypes(nodeTypes: INodeTypeDescription[]): Map { + const versionedTypes = new Map(); + + for (const nt of nodeTypes) { + const versions = Array.isArray(nt.version) + ? nt.version + : [nt.version ?? DEFAULT_NODETYPE_VERSION]; + + const versioned: VersionedTypes = + versionedTypes.get(nt.name) ?? new Map(); + for (const version of versions) { + versioned.set(version, { ...versioned.get(version), ...nt }); + } + + versionedTypes.set(nt.name, versioned); + } + + return versionedTypes; + } + + // This isn't used in Workflow from what I can see + getByName(_nodeType: string): INodeType | IVersionedNodeType { + throw new ApplicationError('Unimplemented `getByName`', { level: 'error' }); + } + + getByNameAndVersion(nodeType: string, version?: number): INodeType { + const versions = this.nodeTypesByVersion.get(nodeType); + if (!versions) { + return undefined as unknown as INodeType; + } + const nodeVersion = versions.get(version ?? Math.max(...versions.keys())); + if (!nodeVersion) { + return undefined as unknown as INodeType; + } + return { + description: nodeVersion, + }; + } + + // This isn't used in Workflow from what I can see + getKnownTypes(): IDataObject { + throw new ApplicationError('Unimplemented `getKnownTypes`', { level: 'error' }); + } +} diff --git a/packages/@n8n/task-runner/src/runner-types.ts b/packages/@n8n/task-runner/src/runner-types.ts index 27b4e9a76c..1e84843653 100644 --- a/packages/@n8n/task-runner/src/runner-types.ts +++ b/packages/@n8n/task-runner/src/runner-types.ts @@ -1,4 +1,4 @@ -import type { INodeExecutionData } from 'n8n-workflow'; +import type { INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow'; export type DataRequestType = 'input' | 'node' | 'all'; @@ -50,6 +50,11 @@ export namespace N8nMessage { data: unknown; } + export interface NodeTypes { + type: 'broker:nodetypes'; + nodeTypes: INodeTypeBaseDescription[]; + } + export type All = | InfoRequest | TaskOfferAccept @@ -57,7 +62,8 @@ export namespace N8nMessage { | TaskSettings | RunnerRegistered | RPCResponse - | TaskDataResponse; + | TaskDataResponse + | NodeTypes; } export namespace ToRequester { diff --git a/packages/@n8n/task-runner/src/start.ts b/packages/@n8n/task-runner/src/start.ts index 5f856140d9..fcaab84d51 100644 --- a/packages/@n8n/task-runner/src/start.ts +++ b/packages/@n8n/task-runner/src/start.ts @@ -1,34 +1,12 @@ -import { ApplicationError, ensureError } from 'n8n-workflow'; -import * as a from 'node:assert/strict'; +import { ensureError } from 'n8n-workflow'; +import Container from 'typedi'; -import { authenticate } from './authenticator'; +import { MainConfig } from './config/main-config'; import { JsTaskRunner } from './js-task-runner/js-task-runner'; let runner: JsTaskRunner | undefined; let isShuttingDown = false; -type Config = { - n8nUri: string; - authToken?: string; - grantToken?: string; -}; - -function readAndParseConfig(): Config { - const authToken = process.env.N8N_RUNNERS_AUTH_TOKEN; - const grantToken = process.env.N8N_RUNNERS_GRANT_TOKEN; - if (!authToken && !grantToken) { - throw new ApplicationError( - 'Missing task runner authentication. Use either N8N_RUNNERS_AUTH_TOKEN or N8N_RUNNERS_GRANT_TOKEN to configure it', - ); - } - - return { - n8nUri: process.env.N8N_RUNNERS_N8N_URI ?? '127.0.0.1:5679', - authToken, - grantToken, - }; -} - function createSignalHandler(signal: string) { return async function onSignal() { if (isShuttingDown) { @@ -53,26 +31,9 @@ function createSignalHandler(signal: string) { } void (async function start() { - const config = readAndParseConfig(); + const config = Container.get(MainConfig); - let grantToken = config.grantToken; - if (!grantToken) { - a.ok(config.authToken); - - grantToken = await authenticate({ - authToken: config.authToken, - n8nUri: config.n8nUri, - }); - } - - const wsUrl = `ws://${config.n8nUri}/runners/_ws`; - runner = new JsTaskRunner({ - wsUrl, - grantToken, - maxConcurrency: 5, - allowedBuiltInModules: process.env.NODE_FUNCTION_ALLOW_BUILTIN, - allowedExternalModules: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, - }); + runner = new JsTaskRunner(config); process.on('SIGINT', createSignalHandler('SIGINT')); process.on('SIGTERM', createSignalHandler('SIGTERM')); diff --git a/packages/@n8n/task-runner/src/task-runner.ts b/packages/@n8n/task-runner/src/task-runner.ts index ac8378636a..9629cc15d5 100644 --- a/packages/@n8n/task-runner/src/task-runner.ts +++ b/packages/@n8n/task-runner/src/task-runner.ts @@ -1,8 +1,9 @@ -import { ApplicationError } from 'n8n-workflow'; +import { ApplicationError, type INodeTypeDescription } from 'n8n-workflow'; import { nanoid } from 'nanoid'; -import { URL } from 'node:url'; import { type MessageEvent, WebSocket } from 'ws'; +import type { BaseRunnerConfig } from './config/base-runner-config'; +import { TaskRunnerNodeTypes } from './node-types'; import { RPC_ALLOW_LIST, type RunnerMessage, @@ -41,6 +42,11 @@ export interface RPCCallObject { const VALID_TIME_MS = 1000; const VALID_EXTRA_MS = 100; +export interface TaskRunnerOpts extends BaseRunnerConfig { + taskType: string; + name?: string; +} + export abstract class TaskRunner { id: string = nanoid(); @@ -58,19 +64,25 @@ export abstract class TaskRunner { rpcCalls: Map = new Map(); - constructor( - public taskType: string, - wsUrl: string, - grantToken: string, - private maxConcurrency: number, - public name?: string, - ) { - const url = new URL(wsUrl); - url.searchParams.append('id', this.id); - this.ws = new WebSocket(url.toString(), { + nodeTypes: TaskRunnerNodeTypes = new TaskRunnerNodeTypes([]); + + taskType: string; + + maxConcurrency: number; + + name: string; + + constructor(opts: TaskRunnerOpts) { + this.taskType = opts.taskType; + this.name = opts.name ?? 'Node.js Task Runner SDK'; + this.maxConcurrency = opts.maxConcurrency; + + const wsUrl = `ws://${opts.n8nUri}/runners/_ws?id=${this.id}`; + this.ws = new WebSocket(wsUrl, { headers: { - authorization: `Bearer ${grantToken}`, + authorization: `Bearer ${opts.grantToken}`, }, + maxPayload: opts.maxPayloadSize, }); this.ws.addEventListener('message', this.receiveMessage); this.ws.addEventListener('close', this.stopTaskOffers); @@ -137,7 +149,7 @@ export abstract class TaskRunner { case 'broker:inforequest': this.send({ type: 'runner:info', - name: this.name ?? 'Node.js Task Runner SDK', + name: this.name, types: [this.taskType], }); break; @@ -158,9 +170,17 @@ export abstract class TaskRunner { break; case 'broker:rpcresponse': this.handleRpcResponse(message.callId, message.status, message.data); + break; + case 'broker:nodetypes': + this.setNodeTypes(message.nodeTypes as unknown as INodeTypeDescription[]); + break; } } + setNodeTypes(nodeTypes: INodeTypeDescription[]) { + this.nodeTypes = new TaskRunnerNodeTypes(nodeTypes); + } + processDataResponse(requestId: string, data: unknown) { const request = this.dataRequests.get(requestId); if (!request) { diff --git a/packages/@n8n/task-runner/tsconfig.json b/packages/@n8n/task-runner/tsconfig.json index db6ad545e3..ddee64ec1f 100644 --- a/packages/@n8n/task-runner/tsconfig.json +++ b/packages/@n8n/task-runner/tsconfig.json @@ -2,6 +2,8 @@ "extends": ["../../../tsconfig.json", "../../../tsconfig.backend.json"], "compilerOptions": { "rootDir": ".", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, "baseUrl": "src", "paths": { "@/*": ["./*"] diff --git a/packages/cli/BREAKING-CHANGES.md b/packages/cli/BREAKING-CHANGES.md index bdb1ff5890..cf29f6d85f 100644 --- a/packages/cli/BREAKING-CHANGES.md +++ b/packages/cli/BREAKING-CHANGES.md @@ -2,6 +2,16 @@ This list shows all the versions which include breaking changes and how to upgrade. +# 1.65.0 + +### What changed? + +Queue polling via the env var `QUEUE_RECOVERY_INTERVAL` has been removed. + +### When is action necessary? + +If you have set the env var `QUEUE_RECOVERY_INTERVAL`, so you can remove it as it no longer has any effect. + # 1.63.0 ### What changed? diff --git a/packages/cli/bin/n8n b/packages/cli/bin/n8n index c4b593ccc9..c3355767af 100755 --- a/packages/cli/bin/n8n +++ b/packages/cli/bin/n8n @@ -18,21 +18,20 @@ if (process.argv.length === 2) { process.argv.push('start'); } -const nodeVersion = process.versions.node; -const { major, gte } = require('semver'); - -const MINIMUM_SUPPORTED_NODE_VERSION = '18.17.0'; -const ENFORCE_MIN_NODE_VERSION = process.env.E2E_TESTS !== 'true'; - -if ( - (ENFORCE_MIN_NODE_VERSION && !gte(nodeVersion, MINIMUM_SUPPORTED_NODE_VERSION)) || - ![18, 20, 22].includes(major(nodeVersion)) -) { - console.log(` - Your Node.js version ${nodeVersion} is currently not supported by n8n. - Please use Node.js v${MINIMUM_SUPPORTED_NODE_VERSION} (recommended), v20, or v22 instead! - `); - process.exit(1); +const ENFORCE_NODE_VERSION_RANGE = process.env.E2E_TESTS !== 'true'; +if (ENFORCE_NODE_VERSION_RANGE) { + const satisfies = require('semver/functions/satisfies'); + const nodeVersion = process.versions.node; + const { + engines: { node: supportedNodeVersions }, + } = require('../package.json'); + if (!satisfies(nodeVersion, supportedNodeVersions)) { + console.error(` +Your Node.js version ${nodeVersion} is currently not supported by n8n. +Please use a Node.js version that satisfies the following version range: ${supportedNodeVersions} +`); + process.exit(1); + } } // Disable nodejs custom inspection across the app diff --git a/packages/cli/jest.config.js b/packages/cli/jest.config.js index 72a653e07d..5420435df2 100644 --- a/packages/cli/jest.config.js +++ b/packages/cli/jest.config.js @@ -14,4 +14,5 @@ module.exports = { ], coveragePathIgnorePatterns: ['/src/databases/migrations/'], testTimeout: 10_000, + prettierPath: null, }; diff --git a/packages/cli/package.json b/packages/cli/package.json index d7d92d96bc..2d95aa1cfa 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "1.63.0", + "version": "1.65.0", "description": "n8n Workflow Automation Tool", "main": "dist/index", "types": "dist/index.d.ts", @@ -28,7 +28,7 @@ "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest", "test:postgres": "N8N_LOG_LEVEL=silent DB_TYPE=postgresdb DB_POSTGRESDB_SCHEMA=alt_schema DB_TABLE_PREFIX=test_ jest --no-coverage", "test:mysql": "N8N_LOG_LEVEL=silent DB_TYPE=mysqldb DB_TABLE_PREFIX=test_ jest --no-coverage", - "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" + "watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\"" }, "bin": { "n8n": "./bin/n8n" @@ -42,7 +42,7 @@ "workflow" ], "engines": { - "node": ">=18.10" + "node": ">=18.17 <= 22" }, "files": [ "bin", diff --git a/packages/cli/src/__tests__/active-executions.test.ts b/packages/cli/src/__tests__/active-executions.test.ts index a8c7b0f18e..5432bf5c4a 100644 --- a/packages/cli/src/__tests__/active-executions.test.ts +++ b/packages/cli/src/__tests__/active-executions.test.ts @@ -108,6 +108,15 @@ describe('ActiveExecutions', () => { expect(activeExecutions.getActiveExecutions().length).toBe(0); }); + test('Should not try to resolve a post-execute promise for an inactive execution', async () => { + // @ts-expect-error Private method + const getExecutionSpy = jest.spyOn(activeExecutions, 'getExecution'); + + activeExecutions.finalizeExecution('inactive-execution-id', mockFullRunData()); + + expect(getExecutionSpy).not.toHaveBeenCalled(); + }); + test('Should resolve post execute promise on removal', async () => { const newExecution = mockExecutionData(); const executionId = await activeExecutions.add(newExecution); diff --git a/packages/cli/src/__tests__/license.test.ts b/packages/cli/src/__tests__/license.test.ts index 67a92b95cd..d33d7c37cf 100644 --- a/packages/cli/src/__tests__/license.test.ts +++ b/packages/cli/src/__tests__/license.test.ts @@ -1,3 +1,4 @@ +import type { GlobalConfig } from '@n8n/config'; import { LicenseManager } from '@n8n_io/license-sdk'; import { mock } from 'jest-mock-extended'; import type { InstanceSettings } from 'n8n-core'; @@ -16,14 +17,16 @@ const MOCK_ACTIVATION_KEY = 'activation-key'; const MOCK_FEATURE_FLAG = 'feat:sharing'; const MOCK_MAIN_PLAN_ID = '1b765dc4-d39d-4ffe-9885-c56dd67c4b26'; -describe('License', () => { - beforeAll(() => { - config.set('license.serverUrl', MOCK_SERVER_URL); - config.set('license.autoRenewEnabled', true); - config.set('license.autoRenewOffset', MOCK_RENEW_OFFSET); - config.set('license.tenantId', 1); - }); +const licenseConfig: GlobalConfig['license'] = { + serverUrl: MOCK_SERVER_URL, + autoRenewalEnabled: true, + autoRenewOffset: MOCK_RENEW_OFFSET, + activationKey: MOCK_ACTIVATION_KEY, + tenantId: 1, + cert: '', +}; +describe('License', () => { let license: License; const instanceSettings = mock({ instanceId: MOCK_INSTANCE_ID, @@ -31,7 +34,11 @@ describe('License', () => { }); beforeEach(async () => { - license = new License(mockLogger(), instanceSettings, mock(), mock(), mock()); + const globalConfig = mock({ + license: licenseConfig, + multiMainSetup: { enabled: false }, + }); + license = new License(mockLogger(), instanceSettings, mock(), mock(), mock(), globalConfig); await license.init(); }); @@ -64,6 +71,7 @@ describe('License', () => { mock(), mock(), mock(), + mock({ license: licenseConfig }), ); await license.init(); expect(LicenseManager).toHaveBeenCalledWith( @@ -189,17 +197,23 @@ describe('License', () => { }); describe('License', () => { - beforeEach(() => { - config.load(config.default); - }); - describe('init', () => { describe('in single-main setup', () => { describe('with `license.autoRenewEnabled` enabled', () => { it('should enable renewal', async () => { - config.set('multiMainSetup.enabled', false); + const globalConfig = mock({ + license: licenseConfig, + multiMainSetup: { enabled: false }, + }); - await new License(mockLogger(), mock(), mock(), mock(), mock()).init(); + await new License( + mockLogger(), + mock({ instanceType: 'main' }), + mock(), + mock(), + mock(), + globalConfig, + ).init(); expect(LicenseManager).toHaveBeenCalledWith( expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), @@ -209,9 +223,14 @@ describe('License', () => { describe('with `license.autoRenewEnabled` disabled', () => { it('should disable renewal', async () => { - config.set('license.autoRenewEnabled', false); - - await new License(mockLogger(), mock(), mock(), mock(), mock()).init(); + await new License( + mockLogger(), + mock({ instanceType: 'main' }), + mock(), + mock(), + mock(), + mock(), + ).init(); expect(LicenseManager).toHaveBeenCalledWith( expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), @@ -225,11 +244,13 @@ describe('License', () => { test.each(['unset', 'leader', 'follower'])( 'if %s status, should disable removal', async (status) => { - config.set('multiMainSetup.enabled', true); + const globalConfig = mock({ + license: { ...licenseConfig, autoRenewalEnabled: false }, + multiMainSetup: { enabled: true }, + }); config.set('multiMainSetup.instanceType', status); - config.set('license.autoRenewEnabled', false); - await new License(mockLogger(), mock(), mock(), mock(), mock()).init(); + await new License(mockLogger(), mock(), mock(), mock(), mock(), globalConfig).init(); expect(LicenseManager).toHaveBeenCalledWith( expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), @@ -240,11 +261,13 @@ describe('License', () => { describe('with `license.autoRenewEnabled` enabled', () => { test.each(['unset', 'follower'])('if %s status, should disable removal', async (status) => { - config.set('multiMainSetup.enabled', true); + const globalConfig = mock({ + license: { ...licenseConfig, autoRenewalEnabled: false }, + multiMainSetup: { enabled: true }, + }); config.set('multiMainSetup.instanceType', status); - config.set('license.autoRenewEnabled', false); - await new License(mockLogger(), mock(), mock(), mock(), mock()).init(); + await new License(mockLogger(), mock(), mock(), mock(), mock(), globalConfig).init(); expect(LicenseManager).toHaveBeenCalledWith( expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), @@ -252,10 +275,13 @@ describe('License', () => { }); it('if leader status, should enable renewal', async () => { - config.set('multiMainSetup.enabled', true); + const globalConfig = mock({ + license: licenseConfig, + multiMainSetup: { enabled: true }, + }); config.set('multiMainSetup.instanceType', 'leader'); - await new License(mockLogger(), mock(), mock(), mock(), mock()).init(); + await new License(mockLogger(), mock(), mock(), mock(), mock(), globalConfig).init(); expect(LicenseManager).toHaveBeenCalledWith( expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), @@ -267,7 +293,7 @@ describe('License', () => { describe('reinit', () => { it('should reinitialize license manager', async () => { - const license = new License(mockLogger(), mock(), mock(), mock(), mock()); + const license = new License(mockLogger(), mock(), mock(), mock(), mock(), mock()); await license.init(); const initSpy = jest.spyOn(license, 'init'); diff --git a/packages/cli/src/__tests__/node-types.test.ts b/packages/cli/src/__tests__/node-types.test.ts new file mode 100644 index 0000000000..11e2c5ba2b --- /dev/null +++ b/packages/cli/src/__tests__/node-types.test.ts @@ -0,0 +1,100 @@ +import { mock } from 'jest-mock-extended'; +import type { INodeType, IVersionedNodeType } from 'n8n-workflow'; + +import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; + +import { NodeTypes } from '../node-types'; + +describe('NodeTypes', () => { + let nodeTypes: NodeTypes; + const loadNodesAndCredentials = mock(); + + beforeEach(() => { + jest.clearAllMocks(); + nodeTypes = new NodeTypes(loadNodesAndCredentials); + }); + + describe('getByNameAndVersion', () => { + const nodeTypeName = 'n8n-nodes-base.testNode'; + + it('should throw an error if the node-type does not exist', () => { + const nodeTypeName = 'unknownNode'; + + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.loadedNodes = {}; + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.knownNodes = {}; + + expect(() => nodeTypes.getByNameAndVersion(nodeTypeName)).toThrow( + 'Unrecognized node type: unknownNode', + ); + }); + + it('should return a regular node-type without version', () => { + const nodeType = mock(); + + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.loadedNodes = { + [nodeTypeName]: { type: nodeType }, + }; + + const result = nodeTypes.getByNameAndVersion(nodeTypeName); + + expect(result).toEqual(nodeType); + }); + + it('should return a regular node-type with version', () => { + const nodeTypeV1 = mock(); + const nodeType = mock({ + nodeVersions: { 1: nodeTypeV1 }, + getNodeType: () => nodeTypeV1, + }); + + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.loadedNodes = { + [nodeTypeName]: { type: nodeType }, + }; + + const result = nodeTypes.getByNameAndVersion(nodeTypeName); + + expect(result).toEqual(nodeTypeV1); + }); + + it('should throw when a node-type is requested as tool, but does not support being used as one', () => { + const nodeType = mock(); + + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.loadedNodes = { + [nodeTypeName]: { type: nodeType }, + }; + + expect(() => nodeTypes.getByNameAndVersion(`${nodeTypeName}Tool`)).toThrow( + 'Node cannot be used as a tool', + ); + }); + + it('should return the tool node-type when requested as tool', () => { + const nodeType = mock(); + // @ts-expect-error can't use a mock here + nodeType.description = { + name: nodeTypeName, + displayName: 'TestNode', + usableAsTool: true, + properties: [], + }; + + // @ts-expect-error overwriting a readonly property + loadNodesAndCredentials.loadedNodes = { + [nodeTypeName]: { type: nodeType }, + }; + + const result = nodeTypes.getByNameAndVersion(`${nodeTypeName}Tool`); + expect(result).not.toEqual(nodeType); + expect(result.description.name).toEqual('n8n-nodes-base.testNodeTool'); + expect(result.description.displayName).toEqual('TestNode Tool'); + expect(result.description.codex?.categories).toContain('AI'); + expect(result.description.inputs).toEqual([]); + expect(result.description.outputs).toEqual(['ai_tool']); + }); + }); +}); diff --git a/packages/cli/src/__tests__/wait-tracker.test.ts b/packages/cli/src/__tests__/wait-tracker.test.ts index 8f9f31da00..66c26f00c6 100644 --- a/packages/cli/src/__tests__/wait-tracker.test.ts +++ b/packages/cli/src/__tests__/wait-tracker.test.ts @@ -1,8 +1,9 @@ import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import type { IExecutionResponse } from '@/interfaces'; -import type { MultiMainSetup } from '@/services/orchestration/main/multi-main-setup.ee'; +import type { MultiMainSetup } from '@/scaling/multi-main-setup.ee'; import { OrchestrationService } from '@/services/orchestration.service'; import { WaitTracker } from '@/wait-tracker'; import { mockLogger } from '@test/mocking'; @@ -12,7 +13,8 @@ jest.useFakeTimers(); describe('WaitTracker', () => { const executionRepository = mock(); const multiMainSetup = mock(); - const orchestrationService = new OrchestrationService(mock(), mock(), multiMainSetup); + const orchestrationService = new OrchestrationService(mock(), multiMainSetup, mock()); + const instanceSettings = mock({ isLeader: true }); const execution = mock({ id: '123', @@ -27,6 +29,7 @@ describe('WaitTracker', () => { mock(), mock(), orchestrationService, + instanceSettings, ); multiMainSetup.on.mockReturnThis(); }); @@ -37,7 +40,6 @@ describe('WaitTracker', () => { describe('init()', () => { it('should query DB for waiting executions if leader', async () => { - jest.spyOn(orchestrationService, 'isLeader', 'get').mockReturnValue(true); executionRepository.getWaitingExecutions.mockResolvedValue([execution]); waitTracker.init(); @@ -120,7 +122,6 @@ describe('WaitTracker', () => { describe('multi-main setup', () => { it('should start tracking if leader', () => { - jest.spyOn(orchestrationService, 'isLeader', 'get').mockReturnValue(true); jest.spyOn(orchestrationService, 'isSingleMainSetup', 'get').mockReturnValue(false); executionRepository.getWaitingExecutions.mockResolvedValue([]); @@ -131,7 +132,14 @@ describe('WaitTracker', () => { }); it('should not start tracking if follower', () => { - jest.spyOn(orchestrationService, 'isLeader', 'get').mockReturnValue(false); + const waitTracker = new WaitTracker( + mockLogger(), + executionRepository, + mock(), + mock(), + orchestrationService, + mock({ isLeader: false }), + ); jest.spyOn(orchestrationService, 'isSingleMainSetup', 'get').mockReturnValue(false); executionRepository.getWaitingExecutions.mockResolvedValue([]); diff --git a/packages/cli/src/abstract-server.ts b/packages/cli/src/abstract-server.ts index 95ecaccdc5..f440b2879a 100644 --- a/packages/cli/src/abstract-server.ts +++ b/packages/cli/src/abstract-server.ts @@ -5,7 +5,6 @@ import { engine as expressHandlebars } from 'express-handlebars'; import { readFile } from 'fs/promises'; import type { Server } from 'http'; import isbot from 'isbot'; -import type { InstanceType } from 'n8n-core'; import { Container, Service } from 'typedi'; import config from '@/config'; @@ -16,13 +15,12 @@ import { ExternalHooks } from '@/external-hooks'; import { Logger } from '@/logging/logger.service'; import { rawBodyReader, bodyParser, corsMiddleware } from '@/middlewares'; import { send, sendErrorResponse } from '@/response-helper'; -import { WaitingForms } from '@/waiting-forms'; import { LiveWebhooks } from '@/webhooks/live-webhooks'; import { TestWebhooks } from '@/webhooks/test-webhooks'; +import { WaitingForms } from '@/webhooks/waiting-forms'; import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; import { createWebhookHandlerFor } from '@/webhooks/webhook-request-handler'; -import { generateHostInstanceId } from './databases/utils/generators'; import { ServiceUnavailableError } from './errors/response-errors/service-unavailable.error'; @Service() @@ -61,7 +59,7 @@ export abstract class AbstractServer { readonly uniqueInstanceId: string; - constructor(instanceType: Exclude) { + constructor() { this.app = express(); this.app.disable('x-powered-by'); @@ -85,8 +83,6 @@ export abstract class AbstractServer { this.endpointWebhookTest = this.globalConfig.endpoints.webhookTest; this.endpointWebhookWaiting = this.globalConfig.endpoints.webhookWaiting; - this.uniqueInstanceId = generateHostInstanceId(instanceType); - this.logger = Container.get(Logger); } diff --git a/packages/cli/src/active-executions.ts b/packages/cli/src/active-executions.ts index f5835ca164..bc18eade16 100644 --- a/packages/cli/src/active-executions.ts +++ b/packages/cli/src/active-executions.ts @@ -154,6 +154,7 @@ export class ActiveExecutions { /** Resolve the post-execution promise in an execution. */ finalizeExecution(executionId: string, fullRunData?: IRun) { + if (!this.has(executionId)) return; const execution = this.getExecution(executionId); execution.postExecutePromise.resolve(fullRunData); this.logger.debug('Execution finalized', { executionId }); diff --git a/packages/cli/src/active-workflow-manager.ts b/packages/cli/src/active-workflow-manager.ts index 9c5ef15f38..189c446b65 100644 --- a/packages/cli/src/active-workflow-manager.ts +++ b/packages/cli/src/active-workflow-manager.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import { ActiveWorkflows, NodeExecuteFunctions } from 'n8n-core'; +import { ActiveWorkflows, InstanceSettings, NodeExecuteFunctions } from 'n8n-core'; import type { ExecutionError, IDeferredPromise, @@ -48,6 +48,7 @@ import { WorkflowExecutionService } from '@/workflows/workflow-execution.service import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; import { ExecutionService } from './executions/execution.service'; +import { Publisher } from './scaling/pubsub/publisher.service'; interface QueuedActivation { activationMode: WorkflowActivateMode; @@ -74,6 +75,8 @@ export class ActiveWorkflowManager { private readonly workflowStaticDataService: WorkflowStaticDataService, private readonly activeWorkflowsService: ActiveWorkflowsService, private readonly workflowExecutionService: WorkflowExecutionService, + private readonly instanceSettings: InstanceSettings, + private readonly publisher: Publisher, ) {} async init() { @@ -423,7 +426,7 @@ export class ActiveWorkflowManager { if (dbWorkflows.length === 0) return; - if (this.orchestrationService.isLeader) { + if (this.instanceSettings.isLeader) { this.logger.info(' ================================'); this.logger.info(' Start Active Workflows:'); this.logger.info(' ================================'); @@ -516,8 +519,9 @@ export class ActiveWorkflowManager { { shouldPublish } = { shouldPublish: true }, ) { if (this.orchestrationService.isMultiMainSetupEnabled && shouldPublish) { - await this.orchestrationService.publish('add-webhooks-triggers-and-pollers', { - workflowId, + void this.publisher.publishCommand({ + command: 'add-webhooks-triggers-and-pollers', + payload: { workflowId }, }); return; @@ -525,8 +529,8 @@ export class ActiveWorkflowManager { let workflow: Workflow; - const shouldAddWebhooks = this.orchestrationService.shouldAddWebhooks(activationMode); - const shouldAddTriggersAndPollers = this.orchestrationService.shouldAddTriggersAndPollers(); + const shouldAddWebhooks = this.shouldAddWebhooks(activationMode); + const shouldAddTriggersAndPollers = this.shouldAddTriggersAndPollers(); const shouldDisplayActivationMessage = (shouldAddWebhooks || shouldAddTriggersAndPollers) && @@ -716,7 +720,10 @@ export class ActiveWorkflowManager { ); } - await this.orchestrationService.publish('remove-triggers-and-pollers', { workflowId }); + void this.publisher.publishCommand({ + command: 'remove-triggers-and-pollers', + payload: { workflowId }, + }); return; } @@ -809,4 +816,29 @@ export class ActiveWorkflowManager { async removeActivationError(workflowId: string) { await this.activationErrorsService.deregister(workflowId); } + + /** + * Whether this instance may add webhooks to the `webhook_entity` table. + */ + shouldAddWebhooks(activationMode: WorkflowActivateMode) { + // Always try to populate the webhook entity table as well as register the webhooks + // to prevent issues with users upgrading from a version < 1.15, where the webhook entity + // was cleared on shutdown to anything past 1.28.0, where we stopped populating it on init, + // causing all webhooks to break + if (activationMode === 'init') return true; + + if (activationMode === 'leadershipChange') return false; + + return this.instanceSettings.isLeader; // 'update' or 'activate' + } + + /** + * Whether this instance may add triggers and pollers to memory. + * + * In both single- and multi-main setup, only the leader is allowed to manage + * triggers and pollers in memory, to ensure they are not duplicated. + */ + shouldAddTriggersAndPollers() { + return this.instanceSettings.isLeader; + } } diff --git a/packages/cli/src/commands/audit.ts b/packages/cli/src/commands/audit.ts index e86c8c9ab5..e98bb8bce0 100644 --- a/packages/cli/src/commands/audit.ts +++ b/packages/cli/src/commands/audit.ts @@ -1,8 +1,8 @@ +import { SecurityConfig } from '@n8n/config'; import { Flags } from '@oclif/core'; import { ApplicationError } from 'n8n-workflow'; import { Container } from 'typedi'; -import config from '@/config'; import { RISK_CATEGORIES } from '@/security-audit/constants'; import { SecurityAuditService } from '@/security-audit/security-audit.service'; import type { Risk } from '@/security-audit/types'; @@ -26,7 +26,7 @@ export class SecurityAudit extends BaseCommand { }), 'days-abandoned-workflow': Flags.integer({ - default: config.getEnv('security.audit.daysAbandonedWorkflow'), + default: Container.get(SecurityConfig).daysAbandonedWorkflow, description: 'Days for a workflow to be considered abandoned if not executed', }), }; diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index f4d97a6a05..214d7f4ce7 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -19,7 +19,6 @@ import type { AbstractServer } from '@/abstract-server'; import config from '@/config'; import { LICENSE_FEATURES, inDevelopment, inTest } from '@/constants'; import * as CrashJournal from '@/crash-journal'; -import { generateHostInstanceId } from '@/databases/utils/generators'; import * as Db from '@/db'; import { getDataDeduplicationService } from '@/deduplication'; import { initErrorHandling } from '@/error-reporting'; @@ -45,8 +44,6 @@ export abstract class BaseCommand extends Command { protected instanceSettings: InstanceSettings = Container.get(InstanceSettings); - queueModeId: string; - protected server?: AbstractServer; protected shutdownService: ShutdownService = Container.get(ShutdownService); @@ -58,7 +55,8 @@ export abstract class BaseCommand extends Command { /** * How long to wait for graceful shutdown before force killing the process. */ - protected gracefulShutdownTimeoutInS = config.getEnv('generic.gracefulShutdownTimeout'); + protected gracefulShutdownTimeoutInS = + Container.get(GlobalConfig).generic.gracefulShutdownTimeout; /** Whether to init community packages (if enabled) */ protected needsCommunityPackages = false; @@ -133,16 +131,6 @@ export abstract class BaseCommand extends Command { await Container.get(TelemetryEventRelay).init(); } - protected setInstanceQueueModeId() { - if (config.get('redis.queueModeId')) { - this.queueModeId = config.get('redis.queueModeId'); - return; - } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - this.queueModeId = generateHostInstanceId(this.instanceSettings.instanceType!); - config.set('redis.queueModeId', this.queueModeId); - } - protected async stopProcess() { // This needs to be overridden } @@ -286,7 +274,7 @@ export abstract class BaseCommand extends Command { this.license = Container.get(License); await this.license.init(); - const activationKey = config.getEnv('license.activationKey'); + const { activationKey } = this.globalConfig.license; if (activationKey) { const hasCert = (await this.license.loadCertStr()).length > 0; diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 428f451fdc..70f52f8cb8 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import { Flags, type Config } from '@oclif/core'; +import { GlobalConfig } from '@n8n/config'; +import { Flags } from '@oclif/core'; import glob from 'fast-glob'; import { createReadStream, createWriteStream, existsSync } from 'fs'; import { mkdir } from 'fs/promises'; @@ -21,7 +22,7 @@ import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus' import { EventService } from '@/events/event.service'; import { ExecutionService } from '@/executions/execution.service'; import { License } from '@/license'; -import { SingleMainTaskManager } from '@/runners/task-managers/single-main-task-manager'; +import { LocalTaskManager } from '@/runners/task-managers/local-task-manager'; import { TaskManager } from '@/runners/task-managers/task-manager'; import { PubSubHandler } from '@/scaling/pubsub/pubsub-handler'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; @@ -70,11 +71,6 @@ export class Start extends BaseCommand { override needsCommunityPackages = true; - constructor(argv: string[], cmdConfig: Config) { - super(argv, cmdConfig); - this.setInstanceQueueModeId(); - } - /** * Opens the UI in browser */ @@ -174,8 +170,9 @@ export class Start extends BaseCommand { this.logger.info('Initializing n8n process'); if (config.getEnv('executions.mode') === 'queue') { - this.logger.debug('Main Instance running in queue mode'); - this.logger.debug(`Queue mode id: ${this.queueModeId}`); + const scopedLogger = this.logger.scoped('scaling'); + scopedLogger.debug('Starting main instance in scaling mode'); + scopedLogger.debug(`Host ID: ${this.instanceSettings.hostId}`); } const { flags } = await this.parse(Start); @@ -202,7 +199,7 @@ export class Start extends BaseCommand { await this.initOrchestration(); this.logger.debug('Orchestration init complete'); - if (!config.getEnv('license.autoRenewEnabled') && this.instanceSettings.isLeader) { + if (!this.globalConfig.license.autoRenewalEnabled && this.instanceSettings.isLeader) { this.logger.warn( 'Automatic license renewal is disabled. The license will not renew automatically, and access to licensed features may be lost!', ); @@ -225,15 +222,21 @@ export class Start extends BaseCommand { await this.generateStaticAssets(); } - if (!this.globalConfig.taskRunners.disabled) { - Container.set(TaskManager, new SingleMainTaskManager()); + const { taskRunners: taskRunnerConfig } = this.globalConfig; + if (!taskRunnerConfig.disabled) { + Container.set(TaskManager, new LocalTaskManager()); const { TaskRunnerServer } = await import('@/runners/task-runner-server'); const taskRunnerServer = Container.get(TaskRunnerServer); await taskRunnerServer.start(); - const { TaskRunnerProcess } = await import('@/runners/task-runner-process'); - const runnerProcess = Container.get(TaskRunnerProcess); - await runnerProcess.start(); + if ( + taskRunnerConfig.mode === 'internal_childprocess' || + taskRunnerConfig.mode === 'internal_launcher' + ) { + const { TaskRunnerProcess } = await import('@/runners/task-runner-process'); + const runnerProcess = Container.get(TaskRunnerProcess); + await runnerProcess.start(); + } } } @@ -244,7 +247,7 @@ export class Start extends BaseCommand { } if ( - config.getEnv('multiMainSetup.enabled') && + Container.get(GlobalConfig).multiMainSetup.enabled && !Container.get(License).isMultipleMainInstancesLicensed() ) { throw new FeatureNotLicensedError(LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES); @@ -260,6 +263,8 @@ export class Start extends BaseCommand { await subscriber.subscribe('n8n.commands'); await subscriber.subscribe('n8n.worker-response'); + this.logger.scoped(['scaling', 'pubsub']).debug('Pubsub setup completed'); + if (!orchestrationService.isMultiMainSetupEnabled) return; orchestrationService.multiMainSetup diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 8c601c7ebc..77ec770aa0 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -1,4 +1,4 @@ -import { Flags, type Config } from '@oclif/core'; +import { Flags } from '@oclif/core'; import { ApplicationError } from 'n8n-workflow'; import { Container } from 'typedi'; @@ -6,7 +6,7 @@ import { ActiveExecutions } from '@/active-executions'; import config from '@/config'; import { PubSubHandler } from '@/scaling/pubsub/pubsub-handler'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; -import { OrchestrationWebhookService } from '@/services/orchestration/webhook/orchestration.webhook.service'; +import { OrchestrationService } from '@/services/orchestration.service'; import { WebhookServer } from '@/webhooks/webhook-server'; import { BaseCommand } from './base-command'; @@ -24,14 +24,6 @@ export class Webhook extends BaseCommand { override needsCommunityPackages = true; - constructor(argv: string[], cmdConfig: Config) { - super(argv, cmdConfig); - if (this.queueModeId) { - this.logger.debug(`Webhook Instance queue mode id: ${this.queueModeId}`); - } - this.setInstanceQueueModeId(); - } - /** * Stops n8n in a graceful way. * Make for example sure that all the webhooks from third party services @@ -71,8 +63,8 @@ export class Webhook extends BaseCommand { await this.initCrashJournal(); this.logger.debug('Crash journal initialized'); - this.logger.info('Initializing n8n webhook process'); - this.logger.debug(`Queue mode id: ${this.queueModeId}`); + this.logger.info('Starting n8n webhook process...'); + this.logger.debug(`Host ID: ${this.instanceSettings.hostId}`); await super.init(); @@ -91,7 +83,7 @@ export class Webhook extends BaseCommand { } async run() { - if (config.getEnv('multiMainSetup.enabled')) { + if (this.globalConfig.multiMainSetup.enabled) { throw new ApplicationError( 'Webhook process cannot be started when multi-main setup is enabled.', ); @@ -100,7 +92,6 @@ export class Webhook extends BaseCommand { const { ScalingService } = await import('@/scaling/scaling.service'); await Container.get(ScalingService).setupQueue(); await this.server.start(); - this.logger.debug(`Webhook listener ID: ${this.server.uniqueInstanceId}`); this.logger.info('Webhook listener waiting for requests.'); // Make sure that the process does not close @@ -112,7 +103,7 @@ export class Webhook extends BaseCommand { } async initOrchestration() { - await Container.get(OrchestrationWebhookService).init(); + await Container.get(OrchestrationService).init(); Container.get(PubSubHandler).init(); await Container.get(Subscriber).subscribe('n8n.commands'); diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 528951be4a..4ba505a9b8 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -1,18 +1,20 @@ import { Flags, type Config } from '@oclif/core'; -import { ApplicationError } from 'n8n-workflow'; import { Container } from 'typedi'; import config from '@/config'; import { N8N_VERSION, inTest } from '@/constants'; +import { WorkerMissingEncryptionKey } from '@/errors/worker-missing-encryption-key.error'; import { EventMessageGeneric } from '@/eventbus/event-message-classes/event-message-generic'; import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { LogStreamingEventRelay } from '@/events/relays/log-streaming.event-relay'; -import { JobProcessor } from '@/scaling/job-processor'; +import { Logger } from '@/logging/logger.service'; +import { LocalTaskManager } from '@/runners/task-managers/local-task-manager'; +import { TaskManager } from '@/runners/task-managers/task-manager'; import { PubSubHandler } from '@/scaling/pubsub/pubsub-handler'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import type { ScalingService } from '@/scaling/scaling.service'; import type { WorkerServerEndpointsConfig } from '@/scaling/worker-server'; -import { OrchestrationWorkerService } from '@/services/orchestration/worker/orchestration.worker.service'; +import { OrchestrationService } from '@/services/orchestration.service'; import { BaseCommand } from './base-command'; @@ -39,8 +41,6 @@ export class Worker extends BaseCommand { scalingService: ScalingService; - jobProcessor: JobProcessor; - override needsCommunityPackages = true; /** @@ -49,27 +49,27 @@ export class Worker extends BaseCommand { * get removed. */ async stopProcess() { - this.logger.info('Stopping n8n...'); + this.logger.info('Stopping worker...'); try { await this.externalHooks?.run('n8n.stop', []); } catch (error) { - await this.exitWithCrash('There was an error shutting down n8n.', error); + await this.exitWithCrash('Error shutting down worker', error); } await this.exitSuccessFully(); } constructor(argv: string[], cmdConfig: Config) { - super(argv, cmdConfig); + if (!process.env.N8N_ENCRYPTION_KEY) throw new WorkerMissingEncryptionKey(); - if (!process.env.N8N_ENCRYPTION_KEY) { - throw new ApplicationError( - 'Missing encryption key. Worker started without the required N8N_ENCRYPTION_KEY env var. More information: https://docs.n8n.io/hosting/configuration/configuration-examples/encryption-key/', - ); + if (config.getEnv('executions.mode') !== 'queue') { + config.set('executions.mode', 'queue'); } - this.setInstanceQueueModeId(); + super(argv, cmdConfig); + + this.logger = Container.get(Logger).scoped('scaling'); } async init() { @@ -84,7 +84,7 @@ export class Worker extends BaseCommand { await this.initCrashJournal(); this.logger.debug('Starting n8n worker...'); - this.logger.debug(`Queue mode id: ${this.queueModeId}`); + this.logger.debug(`Host ID: ${this.instanceSettings.hostId}`); await this.setConcurrency(); await super.init(); @@ -109,15 +109,32 @@ export class Worker extends BaseCommand { new EventMessageGeneric({ eventName: 'n8n.worker.started', payload: { - workerId: this.queueModeId, + workerId: this.instanceSettings.hostId, }, }), ); + + const { taskRunners: taskRunnerConfig } = this.globalConfig; + if (!taskRunnerConfig.disabled) { + Container.set(TaskManager, new LocalTaskManager()); + const { TaskRunnerServer } = await import('@/runners/task-runner-server'); + const taskRunnerServer = Container.get(TaskRunnerServer); + await taskRunnerServer.start(); + + if ( + taskRunnerConfig.mode === 'internal_childprocess' || + taskRunnerConfig.mode === 'internal_launcher' + ) { + const { TaskRunnerProcess } = await import('@/runners/task-runner-process'); + const runnerProcess = Container.get(TaskRunnerProcess); + await runnerProcess.start(); + } + } } async initEventBus() { await Container.get(MessageEventBus).initialize({ - workerId: this.queueModeId, + workerId: this.instanceSettings.hostId, }); Container.get(LogStreamingEventRelay).init(); } @@ -129,10 +146,12 @@ export class Worker extends BaseCommand { * The subscription connection adds a handler to handle the command messages */ async initOrchestration() { - await Container.get(OrchestrationWorkerService).init(); + await Container.get(OrchestrationService).init(); Container.get(PubSubHandler).init(); await Container.get(Subscriber).subscribe('n8n.commands'); + + this.logger.scoped(['scaling', 'pubsub']).debug('Pubsub setup completed'); } async setConcurrency() { @@ -150,8 +169,6 @@ export class Worker extends BaseCommand { await this.scalingService.setupQueue(); this.scalingService.setupWorker(this.concurrency); - - this.jobProcessor = Container.get(JobProcessor); } async run() { diff --git a/packages/cli/src/concurrency/concurrency-control.service.ts b/packages/cli/src/concurrency/concurrency-control.service.ts index cf537870f2..3896daad2e 100644 --- a/packages/cli/src/concurrency/concurrency-control.service.ts +++ b/packages/cli/src/concurrency/concurrency-control.service.ts @@ -33,7 +33,7 @@ export class ConcurrencyControlService { private readonly telemetry: Telemetry, private readonly eventService: EventService, ) { - this.logger = this.logger.withScope('executions'); + this.logger = this.logger.scoped('concurrency'); this.productionLimit = config.getEnv('executions.concurrency.productionLimit'); diff --git a/packages/cli/src/config/index.ts b/packages/cli/src/config/index.ts index 0c799aa8a5..c9e34355ba 100644 --- a/packages/cli/src/config/index.ts +++ b/packages/cli/src/config/index.ts @@ -122,7 +122,7 @@ if (executionProcess === 'own') { } setGlobalState({ - defaultTimezone: config.getEnv('generic.timezone'), + defaultTimezone: Container.get(GlobalConfig).generic.timezone, }); // eslint-disable-next-line import/no-default-export diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 047df9341e..8bece9199a 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -162,33 +162,6 @@ export const schema = { }, }, - generic: { - // The timezone to use. Is important for nodes like "Cron" which start the - // workflow automatically at a specified time. This setting can also be - // overwritten on a per workflow basis in the workflow settings in the - // editor. - timezone: { - doc: 'The timezone to use', - format: '*', - default: 'America/New_York', - env: 'GENERIC_TIMEZONE', - }, - - releaseChannel: { - doc: 'N8N release channel', - format: ['stable', 'beta', 'nightly', 'dev'] as const, - default: 'dev', - env: 'N8N_RELEASE_TYPE', - }, - - gracefulShutdownTimeout: { - doc: 'How long should n8n process wait for components to shut down before exiting the process (seconds)', - format: Number, - default: 30, - env: 'N8N_GRACEFUL_SHUTDOWN_TIMEOUT', - }, - }, - secure_cookie: { doc: 'This sets the `Secure` flag on n8n auth cookie', format: Boolean, @@ -214,29 +187,6 @@ export const schema = { doc: 'Public URL where the editor is accessible. Also used for emails sent from n8n.', }, - security: { - restrictFileAccessTo: { - doc: 'If set only files in that directories can be accessed. Multiple directories can be separated by semicolon (";").', - format: String, - default: '', - env: 'N8N_RESTRICT_FILE_ACCESS_TO', - }, - blockFileAccessToN8nFiles: { - doc: 'If set to true it will block access to all files in the ".n8n" directory, the static cache dir at ~/.cache/n8n/public, and user defined config files.', - format: Boolean, - default: true, - env: 'N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES', - }, - audit: { - daysAbandonedWorkflow: { - doc: 'Days for a workflow to be considered abandoned if not executed', - format: Number, - default: 90, - env: 'N8N_SECURITY_AUDIT_DAYS_ABANDONED_WORKFLOW', - }, - }, - }, - workflowTagsDisabled: { format: Boolean, default: false, @@ -438,45 +388,6 @@ export const schema = { env: 'N8N_DEFAULT_LOCALE', }, - license: { - serverUrl: { - format: String, - default: 'https://license.n8n.io/v1', - env: 'N8N_LICENSE_SERVER_URL', - doc: 'License server url to retrieve license.', - }, - autoRenewEnabled: { - format: Boolean, - default: true, - env: 'N8N_LICENSE_AUTO_RENEW_ENABLED', - doc: 'Whether auto renewal for licenses is enabled.', - }, - autoRenewOffset: { - format: Number, - default: 60 * 60 * 72, // 72 hours - env: 'N8N_LICENSE_AUTO_RENEW_OFFSET', - doc: 'How many seconds before expiry a license should get automatically renewed. ', - }, - activationKey: { - format: String, - default: '', - env: 'N8N_LICENSE_ACTIVATION_KEY', - doc: 'Activation key to initialize license', - }, - tenantId: { - format: Number, - default: 1, - env: 'N8N_LICENSE_TENANT_ID', - doc: 'Tenant id used by the license manager', - }, - cert: { - format: String, - default: '', - env: 'N8N_LICENSE_CERT', - doc: 'Ephemeral license certificate', - }, - }, - hideUsagePage: { format: Boolean, default: false, @@ -491,11 +402,6 @@ export const schema = { default: 'n8n', env: 'N8N_REDIS_KEY_PREFIX', }, - queueModeId: { - doc: 'Unique ID for this n8n instance, is usually set automatically by n8n during startup', - format: String, - default: '', - }, }, /** @@ -569,27 +475,6 @@ export const schema = { }, }, - multiMainSetup: { - enabled: { - doc: 'Whether to enable multi-main setup for queue mode (license required)', - format: Boolean, - default: false, - env: 'N8N_MULTI_MAIN_SETUP_ENABLED', - }, - ttl: { - doc: 'Time to live (in seconds) for leader key in multi-main setup', - format: Number, - default: 10, - env: 'N8N_MULTI_MAIN_SETUP_KEY_TTL', - }, - interval: { - doc: 'Interval (in seconds) for leader check in multi-main setup', - format: Number, - default: 3, - env: 'N8N_MULTI_MAIN_SETUP_CHECK_INTERVAL', - }, - }, - proxy_hops: { format: Number, default: 0, diff --git a/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts b/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts index 81025fb2ca..3f34fc1d2c 100644 --- a/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts @@ -1,66 +1,88 @@ import { mock } from 'jest-mock-extended'; -import { randomString } from 'n8n-workflow'; import { Container } from 'typedi'; import type { ApiKey } from '@/databases/entities/api-key'; import type { User } from '@/databases/entities/user'; -import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; +import { EventService } from '@/events/event.service'; import type { ApiKeysRequest, AuthenticatedRequest } from '@/requests'; -import { API_KEY_PREFIX } from '@/services/public-api-key.service'; +import { PublicApiKeyService } from '@/services/public-api-key.service'; import { mockInstance } from '@test/mocking'; import { ApiKeysController } from '../api-keys.controller'; describe('ApiKeysController', () => { - const apiKeysRepository = mockInstance(ApiKeyRepository); + const publicApiKeyService = mockInstance(PublicApiKeyService); + const eventService = mockInstance(EventService); const controller = Container.get(ApiKeysController); let req: AuthenticatedRequest; beforeAll(() => { - req = mock({ user: mock({ id: '123' }) }); + req = { user: { id: '123' } } as AuthenticatedRequest; }); describe('createAPIKey', () => { it('should create and save an API key', async () => { + // Arrange + const apiKeyData = { id: '123', userId: '123', label: 'My API Key', - apiKey: `${API_KEY_PREFIX}${randomString(42)}`, + apiKey: 'apiKey********', createdAt: new Date(), } as ApiKey; - apiKeysRepository.upsert.mockImplementation(); + const req = mock({ user: mock({ id: '123' }) }); - apiKeysRepository.findOneByOrFail.mockResolvedValue(apiKeyData); + publicApiKeyService.createPublicApiKeyForUser.mockResolvedValue(apiKeyData); + + // Act const newApiKey = await controller.createAPIKey(req); - expect(apiKeysRepository.upsert).toHaveBeenCalled(); + // Assert + + expect(publicApiKeyService.createPublicApiKeyForUser).toHaveBeenCalled(); expect(apiKeyData).toEqual(newApiKey); + expect(eventService.emit).toHaveBeenCalledWith( + 'public-api-key-created', + expect.objectContaining({ user: req.user, publicApi: false }), + ); }); }); describe('getAPIKeys', () => { it('should return the users api keys redacted', async () => { + // Arrange + const apiKeyData = { id: '123', userId: '123', label: 'My API Key', - apiKey: `${API_KEY_PREFIX}${randomString(42)}`, + apiKey: 'apiKey***', createdAt: new Date(), + updatedAt: new Date(), } as ApiKey; - apiKeysRepository.findBy.mockResolvedValue([apiKeyData]); + publicApiKeyService.getRedactedApiKeysForUser.mockResolvedValue([apiKeyData]); + + // Act const apiKeys = await controller.getAPIKeys(req); - expect(apiKeys[0].apiKey).not.toEqual(apiKeyData.apiKey); - expect(apiKeysRepository.findBy).toHaveBeenCalledWith({ userId: req.user.id }); + + // Assert + + expect(apiKeys).toEqual([apiKeyData]); + expect(publicApiKeyService.getRedactedApiKeysForUser).toHaveBeenCalledWith( + expect.objectContaining({ id: req.user.id }), + ); }); }); describe('deleteAPIKey', () => { it('should delete the API key', async () => { + // Arrange + const user = mock({ id: '123', password: 'password', @@ -68,12 +90,22 @@ describe('ApiKeysController', () => { role: 'global:member', mfaEnabled: false, }); + const req = mock({ user, params: { id: user.id } }); + + // Act + await controller.deleteAPIKey(req); - expect(apiKeysRepository.delete).toHaveBeenCalledWith({ - userId: req.user.id, - id: req.params.id, - }); + + publicApiKeyService.deleteApiKeyForUser.mockResolvedValue(); + + // Assert + + expect(publicApiKeyService.deleteApiKeyForUser).toHaveBeenCalledWith(user, user.id); + expect(eventService.emit).toHaveBeenCalledWith( + 'public-api-key-deleted', + expect.objectContaining({ user, publicApi: false }), + ); }); }); }); diff --git a/packages/cli/src/controllers/debug.controller.ts b/packages/cli/src/controllers/debug.controller.ts index 9fd2b067d3..1a2b08d550 100644 --- a/packages/cli/src/controllers/debug.controller.ts +++ b/packages/cli/src/controllers/debug.controller.ts @@ -1,3 +1,5 @@ +import { InstanceSettings } from 'n8n-core'; + import { ActiveWorkflowManager } from '@/active-workflow-manager'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { Get, RestController } from '@/decorators'; @@ -9,6 +11,7 @@ export class DebugController { private readonly orchestrationService: OrchestrationService, private readonly activeWorkflowManager: ActiveWorkflowManager, private readonly workflowRepository: WorkflowRepository, + private readonly instanceSettings: InstanceSettings, ) {} @Get('/multi-main-setup', { skipAuth: true }) @@ -24,9 +27,9 @@ export class DebugController { const activationErrors = await this.activeWorkflowManager.getAllWorkflowActivationErrors(); return { - instanceId: this.orchestrationService.instanceId, + instanceId: this.instanceSettings.instanceId, leaderKey, - isLeader: this.orchestrationService.isLeader, + isLeader: this.instanceSettings.isLeader, activeWorkflows: { webhooks, // webhook-based active workflows triggersAndPollers, // poller- and trigger-based active workflows diff --git a/packages/cli/src/controllers/mfa.controller.ts b/packages/cli/src/controllers/mfa.controller.ts index f0af103265..694765761c 100644 --- a/packages/cli/src/controllers/mfa.controller.ts +++ b/packages/cli/src/controllers/mfa.controller.ts @@ -1,11 +1,21 @@ import { Get, Post, RestController } from '@/decorators'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import { ExternalHooks } from '@/external-hooks'; import { MfaService } from '@/mfa/mfa.service'; import { AuthenticatedRequest, MFA } from '@/requests'; @RestController('/mfa') export class MFAController { - constructor(private mfaService: MfaService) {} + constructor( + private mfaService: MfaService, + private externalHooks: ExternalHooks, + ) {} + + @Post('/can-enable') + async canEnableMFA(req: AuthenticatedRequest) { + await this.externalHooks.run('mfa.beforeSetup', [req.user]); + return; + } @Get('/qr') async getQRCode(req: AuthenticatedRequest) { @@ -52,6 +62,8 @@ export class MFAController { const { token = null } = req.body; const { id, mfaEnabled } = req.user; + await this.externalHooks.run('mfa.beforeSetup', [req.user]); + const { decryptedSecret: secret, decryptedRecoveryCodes: recoveryCodes } = await this.mfaService.getSecretAndRecoveryCodes(id); diff --git a/packages/cli/src/controllers/orchestration.controller.ts b/packages/cli/src/controllers/orchestration.controller.ts index db1d690a3e..14d38cfa43 100644 --- a/packages/cli/src/controllers/orchestration.controller.ts +++ b/packages/cli/src/controllers/orchestration.controller.ts @@ -1,31 +1,23 @@ import { Post, RestController, GlobalScope } from '@/decorators'; import { License } from '@/license'; -import { OrchestrationRequest } from '@/requests'; -import { OrchestrationService } from '@/services/orchestration.service'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; @RestController('/orchestration') export class OrchestrationController { constructor( - private readonly orchestrationService: OrchestrationService, private readonly licenseService: License, + private readonly publisher: Publisher, ) {} /** - * These endpoints do not return anything, they just trigger the message to + * This endpoint does not return anything, it just triggers the message to * the workers to respond on Redis with their status. */ - @GlobalScope('orchestration:read') - @Post('/worker/status/:id') - async getWorkersStatus(req: OrchestrationRequest.Get) { - if (!this.licenseService.isWorkerViewLicensed()) return; - const id = req.params.id; - return await this.orchestrationService.getWorkerStatus(id); - } - @GlobalScope('orchestration:read') @Post('/worker/status') async getWorkersStatusAll() { if (!this.licenseService.isWorkerViewLicensed()) return; - return await this.orchestrationService.getWorkerStatus(); + + return await this.publisher.publishCommand({ command: 'get-worker-status' }); } } diff --git a/packages/cli/src/databases/migrations/common/1724753530828-CreateExecutionAnnotationTables.ts b/packages/cli/src/databases/migrations/common/1724753530828-CreateExecutionAnnotationTables.ts index 94c5e5ff68..58cd3cc3f1 100644 --- a/packages/cli/src/databases/migrations/common/1724753530828-CreateExecutionAnnotationTables.ts +++ b/packages/cli/src/databases/migrations/common/1724753530828-CreateExecutionAnnotationTables.ts @@ -25,7 +25,10 @@ export class CreateAnnotationTables1724753530828 implements ReversibleMigration .withIndexOn('name', true).withTimestamps; await createTable(annotationTagMappingsTableName) - .withColumns(column('annotationId').int.notNull, column('tagId').varchar(24).notNull) + .withColumns( + column('annotationId').int.notNull.primary, + column('tagId').varchar(24).notNull.primary, + ) .withForeignKey('annotationId', { tableName: annotationsTableName, columnName: 'id', diff --git a/packages/cli/src/databases/migrations/common/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts b/packages/cli/src/databases/migrations/common/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts new file mode 100644 index 0000000000..51735d660a --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert'; + +import type { IrreversibleMigration, MigrationContext } from '@/databases/types'; + +export class AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644 + implements IrreversibleMigration +{ + async up({ queryRunner, tablePrefix }: MigrationContext) { + // Check if the primary key already exists + const table = await queryRunner.getTable(`${tablePrefix}execution_annotation_tags`); + + assert(table, 'execution_annotation_tags table not found'); + + const hasPrimaryKey = table.primaryColumns.length > 0; + + if (!hasPrimaryKey) { + await queryRunner.createPrimaryKey(`${tablePrefix}execution_annotation_tags`, [ + 'annotationId', + 'tagId', + ]); + } + } +} diff --git a/packages/cli/src/databases/migrations/common/1729607673464-UpdateProcessedDataValueColumnToText.ts b/packages/cli/src/databases/migrations/common/1729607673464-UpdateProcessedDataValueColumnToText.ts new file mode 100644 index 0000000000..074b710fa1 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1729607673464-UpdateProcessedDataValueColumnToText.ts @@ -0,0 +1,34 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const processedDataTableName = 'processed_data'; +export class UpdateProcessedDataValueColumnToText1729607673464 implements ReversibleMigration { + async up({ schemaBuilder: { addNotNull }, isMysql, runQuery, tablePrefix }: MigrationContext) { + const prefixedTableName = `${tablePrefix}${processedDataTableName}`; + await runQuery(`ALTER TABLE ${prefixedTableName} ADD COLUMN value_temp TEXT;`); + await runQuery(`UPDATE ${prefixedTableName} SET value_temp = value;`); + await runQuery(`ALTER TABLE ${prefixedTableName} DROP COLUMN value;`); + + if (isMysql) { + await runQuery(`ALTER TABLE ${prefixedTableName} CHANGE value_temp value TEXT NOT NULL;`); + } else { + await runQuery(`ALTER TABLE ${prefixedTableName} RENAME COLUMN value_temp TO value`); + await addNotNull(processedDataTableName, 'value'); + } + } + + async down({ schemaBuilder: { addNotNull }, isMysql, runQuery, tablePrefix }: MigrationContext) { + const prefixedTableName = `${tablePrefix}${processedDataTableName}`; + await runQuery(`ALTER TABLE ${prefixedTableName} ADD COLUMN value_temp VARCHAR(255);`); + await runQuery(`UPDATE ${prefixedTableName} SET value_temp = value;`); + await runQuery(`ALTER TABLE ${prefixedTableName} DROP COLUMN value;`); + + if (isMysql) { + await runQuery( + `ALTER TABLE ${prefixedTableName} CHANGE value_temp value VARCHAR(255) NOT NULL;`, + ); + } else { + await runQuery(`ALTER TABLE ${prefixedTableName} RENAME COLUMN value_temp TO value`); + await addNotNull(processedDataTableName, 'value'); + } + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 1dcca1e592..ff40fd9dc0 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -66,6 +66,8 @@ import { CreateAnnotationTables1724753530828 } from '../common/1724753530828-Cre import { AddApiKeysTable1724951148974 } from '../common/1724951148974-AddApiKeysTable'; import { CreateProcessedDataTable1726606152711 } from '../common/1726606152711-CreateProcessedDataTable'; import { SeparateExecutionCreationFromStart1727427440136 } from '../common/1727427440136-SeparateExecutionCreationFromStart'; +import { AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644 } from '../common/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping'; +import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/1729607673464-UpdateProcessedDataValueColumnToText'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -134,4 +136,6 @@ export const mysqlMigrations: Migration[] = [ AddApiKeysTable1724951148974, SeparateExecutionCreationFromStart1727427440136, CreateProcessedDataTable1726606152711, + AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644, + UpdateProcessedDataValueColumnToText1729607673464, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index eb0e2bd946..f3ac7e0474 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -66,6 +66,8 @@ import { CreateAnnotationTables1724753530828 } from '../common/1724753530828-Cre import { AddApiKeysTable1724951148974 } from '../common/1724951148974-AddApiKeysTable'; import { CreateProcessedDataTable1726606152711 } from '../common/1726606152711-CreateProcessedDataTable'; import { SeparateExecutionCreationFromStart1727427440136 } from '../common/1727427440136-SeparateExecutionCreationFromStart'; +import { AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644 } from '../common/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping'; +import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/1729607673464-UpdateProcessedDataValueColumnToText'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -134,4 +136,6 @@ export const postgresMigrations: Migration[] = [ AddApiKeysTable1724951148974, SeparateExecutionCreationFromStart1727427440136, CreateProcessedDataTable1726606152711, + AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644, + UpdateProcessedDataValueColumnToText1729607673464, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts b/packages/cli/src/databases/migrations/sqlite/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts new file mode 100644 index 0000000000..1a2900b7a6 --- /dev/null +++ b/packages/cli/src/databases/migrations/sqlite/1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert'; + +import type { IrreversibleMigration, MigrationContext } from '@/databases/types'; + +const annotationsTableName = 'execution_annotations'; +const annotationTagsTableName = 'annotation_tag_entity'; +const annotationTagMappingsTableName = 'execution_annotation_tags'; + +export class AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644 + implements IrreversibleMigration +{ + async up({ + queryRunner, + tablePrefix, + schemaBuilder: { createTable, column, dropIndex }, + }: MigrationContext) { + // Check if the primary key already exists + const table = await queryRunner.getTable(`${tablePrefix}execution_annotation_tags`); + + assert(table, 'execution_annotation_tags table not found'); + + const hasPrimaryKey = table.primaryColumns.length > 0; + + // Do nothing if the primary key already exists + if (hasPrimaryKey) { + return; + } + + // SQLite doesn't support adding a primary key to an existing table + // So we have to do the following steps: + + // 1. Rename the existing table + await queryRunner.query( + `ALTER TABLE ${tablePrefix}${annotationTagMappingsTableName} RENAME TO ${tablePrefix}${annotationTagMappingsTableName}_tmp;`, + ); + + // 1.1 Drop the existing indices + await dropIndex(`${annotationTagMappingsTableName}_tmp`, ['tagId'], { + customIndexName: 'IDX_a3697779b366e131b2bbdae297', + }); + await dropIndex(`${annotationTagMappingsTableName}_tmp`, ['annotationId'], { + customIndexName: 'IDX_c1519757391996eb06064f0e7c', + }); + + // 2. Create a new table with the desired structure + await createTable(annotationTagMappingsTableName) + .withColumns( + column('annotationId').int.notNull.primary, + column('tagId').varchar(24).notNull.primary, + ) + .withForeignKey('annotationId', { + tableName: annotationsTableName, + columnName: 'id', + onDelete: 'CASCADE', + }) + .withIndexOn('tagId') + .withIndexOn('annotationId') + .withForeignKey('tagId', { + tableName: annotationTagsTableName, + columnName: 'id', + onDelete: 'CASCADE', + }); + + // 3. Copy data from the old table to the new one + await queryRunner.query( + `INSERT INTO ${tablePrefix}${annotationTagMappingsTableName} SELECT * FROM ${tablePrefix}${annotationTagMappingsTableName}_tmp;`, + ); + + // 4. Drop the old table + await queryRunner.dropTable(`${tablePrefix}${annotationTagMappingsTableName}_tmp`, true); + } +} diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 797b26752c..e53b5f43bd 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -38,6 +38,7 @@ import { ExecutionSoftDelete1693491613982 } from './1693491613982-ExecutionSoftD import { DropRoleMapping1705429061930 } from './1705429061930-DropRoleMapping'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; import { AddApiKeysTable1724951148974 } from './1724951148974-AddApiKeysTable'; +import { AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644 } from './1728659839644-AddMissingPrimaryKeyOnAnnotationTagMapping'; import { UniqueWorkflowNames1620821879465 } from '../common/1620821879465-UniqueWorkflowNames'; import { UpdateWorkflowCredentials1630330987096 } from '../common/1630330987096-UpdateWorkflowCredentials'; import { AddNodeIds1658930531669 } from '../common/1658930531669-AddNodeIds'; @@ -63,6 +64,7 @@ import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-R import { CreateAnnotationTables1724753530828 } from '../common/1724753530828-CreateExecutionAnnotationTables'; import { CreateProcessedDataTable1726606152711 } from '../common/1726606152711-CreateProcessedDataTable'; import { SeparateExecutionCreationFromStart1727427440136 } from '../common/1727427440136-SeparateExecutionCreationFromStart'; +import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/1729607673464-UpdateProcessedDataValueColumnToText'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -128,6 +130,8 @@ const sqliteMigrations: Migration[] = [ AddApiKeysTable1724951148974, SeparateExecutionCreationFromStart1727427440136, CreateProcessedDataTable1726606152711, + AddMissingPrimaryKeyOnAnnotationTagMapping1728659839644, + UpdateProcessedDataValueColumnToText1729607673464, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/execution-data.repository.ts b/packages/cli/src/databases/repositories/execution-data.repository.ts index 7f54a6f214..f7de742941 100644 --- a/packages/cli/src/databases/repositories/execution-data.repository.ts +++ b/packages/cli/src/databases/repositories/execution-data.repository.ts @@ -1,4 +1,6 @@ import { DataSource, In, Repository } from '@n8n/typeorm'; +import type { EntityManager } from '@n8n/typeorm'; +import type { QueryDeepPartialEntity } from '@n8n/typeorm/query-builder/QueryPartialEntity'; import { Service } from 'typedi'; import { ExecutionData } from '../entities/execution-data'; @@ -9,6 +11,13 @@ export class ExecutionDataRepository extends Repository { super(ExecutionData, dataSource.manager); } + async createExecutionDataForExecution( + data: QueryDeepPartialEntity, + transactionManager: EntityManager, + ) { + return await transactionManager.insert(ExecutionData, data); + } + async findByExecutionIds(executionIds: string[]) { return await this.find({ select: ['workflowData'], diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 7b26463969..ce4bedac2a 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -304,16 +304,34 @@ export class ExecutionRepository extends Repository { * Insert a new execution and its execution data using a transaction. */ async createNewExecution(execution: CreateExecutionPayload): Promise { - const { data, workflowData, ...rest } = execution; - const { identifiers: inserted } = await this.insert({ ...rest, createdAt: new Date() }); - const { id: executionId } = inserted[0] as { id: string }; - const { connections, nodes, name, settings } = workflowData ?? {}; - await this.executionDataRepository.insert({ - executionId, - workflowData: { connections, nodes, name, settings, id: workflowData.id }, - data: stringify(data), - }); - return String(executionId); + const { data: dataObj, workflowData: currentWorkflow, ...rest } = execution; + const { connections, nodes, name, settings } = currentWorkflow ?? {}; + const workflowData = { connections, nodes, name, settings, id: currentWorkflow.id }; + const data = stringify(dataObj); + + const { type: dbType, sqlite: sqliteConfig } = this.globalConfig.database; + if (dbType === 'sqlite' && sqliteConfig.poolSize === 0) { + // TODO: Delete this block of code once the sqlite legacy (non-pooling) driver is dropped. + // In the non-pooling sqlite driver we can't use transactions, because that creates nested transactions under highly concurrent loads, leading to errors in the database + const { identifiers: inserted } = await this.insert({ ...rest, createdAt: new Date() }); + const { id: executionId } = inserted[0] as { id: string }; + await this.executionDataRepository.insert({ executionId, workflowData, data }); + return String(executionId); + } else { + // All other database drivers should create executions and execution-data atomically + return await this.manager.transaction(async (transactionManager) => { + const { identifiers: inserted } = await transactionManager.insert(ExecutionEntity, { + ...rest, + createdAt: new Date(), + }); + const { id: executionId } = inserted[0] as { id: string }; + await this.executionDataRepository.createExecutionDataForExecution( + { executionId, workflowData, data }, + transactionManager, + ); + return String(executionId); + }); + } } async markAsCrashed(executionIds: string | string[]) { diff --git a/packages/cli/src/environments/source-control/__tests__/source-control.service.test.ts b/packages/cli/src/environments/source-control/__tests__/source-control.service.test.ts index 06dab4f97c..e1ebf0e56a 100644 --- a/packages/cli/src/environments/source-control/__tests__/source-control.service.test.ts +++ b/packages/cli/src/environments/source-control/__tests__/source-control.service.test.ts @@ -6,7 +6,7 @@ import { SourceControlService } from '@/environments/source-control/source-contr describe('SourceControlService', () => { const preferencesService = new SourceControlPreferencesService( - new InstanceSettings(), + new InstanceSettings(mock()), mock(), mock(), ); diff --git a/packages/cli/src/error-reporting.ts b/packages/cli/src/error-reporting.ts index e429bdbd30..897bef6fef 100644 --- a/packages/cli/src/error-reporting.ts +++ b/packages/cli/src/error-reporting.ts @@ -3,6 +3,7 @@ import { GlobalConfig } from '@n8n/config'; import { QueryFailedError } from '@n8n/typeorm'; import { AxiosError } from 'axios'; import { createHash } from 'crypto'; +import { InstanceSettings } from 'n8n-core'; import { ErrorReporterProxy, ApplicationError } from 'n8n-workflow'; import Container from 'typedi'; @@ -30,7 +31,7 @@ export const initErrorHandling = async () => { DEPLOYMENT_NAME: serverName, } = process.env; - const { init, captureException } = await import('@sentry/node'); + const { init, captureException, setTag } = await import('@sentry/node'); const { RewriteFrames } = await import('@sentry/integrations'); const { Integrations } = await import('@sentry/node'); @@ -65,9 +66,13 @@ export const initErrorHandling = async () => { }, }), ], - beforeSend(event, { originalException }) { + async beforeSend(event, { originalException }) { if (!originalException) return null; + if (originalException instanceof Promise) { + originalException = await originalException.catch((error) => error as Error); + } + if (originalException instanceof AxiosError) return null; if ( @@ -95,6 +100,8 @@ export const initErrorHandling = async () => { }, }); + setTag('server_type', Container.get(InstanceSettings).instanceType); + ErrorReporterProxy.init({ report: (error, options) => captureException(error, options), }); diff --git a/packages/cli/src/errors/worker-missing-encryption-key.error.ts b/packages/cli/src/errors/worker-missing-encryption-key.error.ts new file mode 100644 index 0000000000..29b8dad929 --- /dev/null +++ b/packages/cli/src/errors/worker-missing-encryption-key.error.ts @@ -0,0 +1,14 @@ +import { ApplicationError } from 'n8n-workflow'; + +export class WorkerMissingEncryptionKey extends ApplicationError { + constructor() { + super( + [ + 'Failed to start worker because of missing encryption key.', + 'Please set the `N8N_ENCRYPTION_KEY` env var when starting the worker.', + 'See: https://docs.n8n.io/hosting/configuration/configuration-examples/encryption-key/', + ].join(' '), + { level: 'warning' }, + ); + } +} diff --git a/packages/cli/src/eventbus/message-event-bus/message-event-bus.ts b/packages/cli/src/eventbus/message-event-bus/message-event-bus.ts index 0f622c2317..3cf5a5a5d0 100644 --- a/packages/cli/src/eventbus/message-event-bus/message-event-bus.ts +++ b/packages/cli/src/eventbus/message-event-bus/message-event-bus.ts @@ -14,7 +14,7 @@ import { ExecutionRepository } from '@/databases/repositories/execution.reposito import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { License } from '@/license'; import { Logger } from '@/logging/logger.service'; -import { OrchestrationService } from '@/services/orchestration.service'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { ExecutionRecoveryService } from '../../executions/execution-recovery.service'; import type { EventMessageTypes } from '../event-message-classes/'; @@ -70,7 +70,7 @@ export class MessageEventBus extends EventEmitter { private readonly executionRepository: ExecutionRepository, private readonly eventDestinationsRepository: EventDestinationsRepository, private readonly workflowRepository: WorkflowRepository, - private readonly orchestrationService: OrchestrationService, + private readonly publisher: Publisher, private readonly recoveryService: ExecutionRecoveryService, private readonly license: License, private readonly globalConfig: GlobalConfig, @@ -210,7 +210,7 @@ export class MessageEventBus extends EventEmitter { this.destinations[destination.getId()] = destination; this.destinations[destination.getId()].startListening(); if (notifyWorkers) { - await this.orchestrationService.publish('restart-event-bus'); + void this.publisher.publishCommand({ command: 'restart-event-bus' }); } return destination; } @@ -236,7 +236,7 @@ export class MessageEventBus extends EventEmitter { delete this.destinations[id]; } if (notifyWorkers) { - await this.orchestrationService.publish('restart-event-bus'); + void this.publisher.publishCommand({ command: 'restart-event-bus' }); } return result; } diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index df65a70ecb..7e98877dc7 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -1061,6 +1061,7 @@ describe('TelemetryEventRelay', () => { describe('Community+ registered', () => { it('should track `license-community-plus-registered` event', () => { const event: RelayEventMap['license-community-plus-registered'] = { + userId: 'user123', email: 'user@example.com', licenseKey: 'license123', }; @@ -1068,6 +1069,7 @@ describe('TelemetryEventRelay', () => { eventService.emit('license-community-plus-registered', event); expect(telemetry.track).toHaveBeenCalledWith('User registered for license community plus', { + user_id: 'user123', email: 'user@example.com', licenseKey: 'license123', }); diff --git a/packages/cli/src/events/maps/relay.event-map.ts b/packages/cli/src/events/maps/relay.event-map.ts index 21b673a2b5..0e72564571 100644 --- a/packages/cli/src/events/maps/relay.event-map.ts +++ b/packages/cli/src/events/maps/relay.event-map.ts @@ -8,7 +8,7 @@ import type { import type { AuthProviderType } from '@/databases/entities/auth-identity'; import type { ProjectRole } from '@/databases/entities/project-relation'; -import type { GlobalRole } from '@/databases/entities/user'; +import type { GlobalRole, User } from '@/databases/entities/user'; import type { IWorkflowDb } from '@/interfaces'; import type { AiEventMap } from './ai.event-map'; @@ -421,6 +421,7 @@ export type RelayEventMap = { }; 'license-community-plus-registered': { + userId: User['id']; email: string; licenseKey: string; }; diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index 11d84751d0..fc5cf0a53d 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -236,10 +236,12 @@ export class TelemetryEventRelay extends EventRelay { } private licenseCommunityPlusRegistered({ + userId, email, licenseKey, }: RelayEventMap['license-community-plus-registered']) { this.telemetry.track('User registered for license community plus', { + user_id: userId, email, licenseKey, }); @@ -651,7 +653,9 @@ export class TelemetryEventRelay extends EventRelay { } if (telemetryProperties.is_manual) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, { + runData: runData.data.resultData?.runData, + }); telemetryProperties.node_graph = nodeGraphResult.nodeGraph; telemetryProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); @@ -663,7 +667,9 @@ export class TelemetryEventRelay extends EventRelay { if (telemetryProperties.is_manual) { if (!nodeGraphResult) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, { + runData: runData.data.resultData?.runData, + }); } let userRole: 'owner' | 'sharee' | undefined = undefined; @@ -688,7 +694,9 @@ export class TelemetryEventRelay extends EventRelay { }; if (!manualExecEventProperties.node_graph_string) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, { + runData: runData.data.resultData?.runData, + }); manualExecEventProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); } @@ -772,9 +780,9 @@ export class TelemetryEventRelay extends EventRelay { ldap_allowed: authenticationMethod === 'ldap', saml_enabled: authenticationMethod === 'saml', license_plan_name: this.license.getPlanName(), - license_tenant_id: config.getEnv('license.tenantId'), + license_tenant_id: this.globalConfig.license.tenantId, binary_data_s3: isS3Available && isS3Selected && isS3Licensed, - multi_main_setup_enabled: config.getEnv('multiMainSetup.enabled'), + multi_main_setup_enabled: this.globalConfig.multiMainSetup.enabled, metrics: { metrics_enabled: this.globalConfig.endpoints.metrics.enable, metrics_category_default: this.globalConfig.endpoints.metrics.includeDefaultMetrics, diff --git a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts index de7c27a8e8..115a1a52f6 100644 --- a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts +++ b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts @@ -22,7 +22,7 @@ import { setupMessages } from './utils'; describe('ExecutionRecoveryService', () => { const push = mockInstance(Push); - const instanceSettings = new InstanceSettings(); + const instanceSettings = new InstanceSettings(mock()); let executionRecoveryService: ExecutionRecoveryService; let executionRepository: ExecutionRepository; diff --git a/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts b/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts index 97547ecf13..05eabd104f 100644 --- a/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts +++ b/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts @@ -13,7 +13,7 @@ import { FailedProvider, MockProviders, } from '@test/external-secrets/utils'; -import { mockInstance } from '@test/mocking'; +import { mockInstance, mockLogger } from '@test/mocking'; describe('External Secrets Manager', () => { const connectedDate = '2023-08-01T12:32:29.000Z'; @@ -49,12 +49,13 @@ describe('External Secrets Manager', () => { license.isExternalSecretsEnabled.mockReturnValue(true); settingsRepo.getEncryptedSecretsProviderSettings.mockResolvedValue(settings); manager = new ExternalSecretsManager( - mock(), + mockLogger(), settingsRepo, license, providersMock, cipher, mock(), + mock(), ); }); diff --git a/packages/cli/src/external-secrets/external-secrets-manager.ee.ts b/packages/cli/src/external-secrets/external-secrets-manager.ee.ts index e175f2969c..2de681a7d6 100644 --- a/packages/cli/src/external-secrets/external-secrets-manager.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets-manager.ee.ts @@ -1,6 +1,6 @@ import { Cipher } from 'n8n-core'; -import { jsonParse, type IDataObject, ApplicationError } from 'n8n-workflow'; -import Container, { Service } from 'typedi'; +import { jsonParse, type IDataObject, ApplicationError, ensureError } from 'n8n-workflow'; +import { Service } from 'typedi'; import { SettingsRepository } from '@/databases/repositories/settings.repository'; import { EventService } from '@/events/event.service'; @@ -11,7 +11,7 @@ import type { } from '@/interfaces'; import { License } from '@/license'; import { Logger } from '@/logging/logger.service'; -import { OrchestrationService } from '@/services/orchestration.service'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { EXTERNAL_SECRETS_INITIAL_BACKOFF, EXTERNAL_SECRETS_MAX_BACKOFF } from './constants'; import { updateIntervalTime } from './external-secrets-helper.ee'; @@ -38,7 +38,10 @@ export class ExternalSecretsManager { private readonly secretsProviders: ExternalSecretsProviders, private readonly cipher: Cipher, private readonly eventService: EventService, - ) {} + private readonly publisher: Publisher, + ) { + this.logger = this.logger.scoped('external-secrets'); + } async init(): Promise { if (!this.initialized) { @@ -56,6 +59,8 @@ export class ExternalSecretsManager { } return await this.initializingPromise; } + + this.logger.debug('External secrets manager initialized'); } shutdown() { @@ -65,6 +70,8 @@ export class ExternalSecretsManager { void p.disconnect().catch(() => {}); }); Object.values(this.initRetryTimeouts).forEach((v) => clearTimeout(v)); + + this.logger.debug('External secrets manager shut down'); } async reloadAllProviders(backoff?: number) { @@ -76,10 +83,12 @@ export class ExternalSecretsManager { for (const provider of providers) { await this.reloadProvider(provider, backoff); } + + this.logger.debug('External secrets managed reloaded all providers'); } - async broadcastReloadExternalSecretsProviders() { - await Container.get(OrchestrationService).publish('reload-external-secrets-providers'); + broadcastReloadExternalSecretsProviders() { + void this.publisher.publishCommand({ command: 'reload-external-secrets-providers' }); } private decryptSecretsSettings(value: string): ExternalSecretsSettings { @@ -190,6 +199,8 @@ export class ExternalSecretsManager { } }), ); + + this.logger.debug('External secrets manager updated secrets'); } getProvider(provider: string): SecretsProvider | undefined { @@ -260,6 +271,8 @@ export class ExternalSecretsManager { if (newProvider) { this.providers[provider] = newProvider; } + + this.logger.debug(`External secrets manager reloaded provider ${provider}`); } async setProviderSettings(provider: string, data: IDataObject, userId?: string) { @@ -280,7 +293,7 @@ export class ExternalSecretsManager { await this.saveAndSetSettings(settings, this.settingsRepo); this.cachedSettings = settings; await this.reloadProvider(provider); - await this.broadcastReloadExternalSecretsProviders(); + this.broadcastReloadExternalSecretsProviders(); void this.trackProviderSave(provider, isNewProvider, userId); } @@ -300,7 +313,7 @@ export class ExternalSecretsManager { this.cachedSettings = settings; await this.reloadProvider(provider); await this.updateSecrets(); - await this.broadcastReloadExternalSecretsProviders(); + this.broadcastReloadExternalSecretsProviders(); } private async trackProviderSave(vaultType: string, isNew: boolean, userId?: string) { @@ -380,9 +393,13 @@ export class ExternalSecretsManager { } try { await this.providers[provider].update(); - await this.broadcastReloadExternalSecretsProviders(); + this.broadcastReloadExternalSecretsProviders(); + this.logger.debug(`External secrets manager updated provider ${provider}`); return true; - } catch { + } catch (error) { + this.logger.debug(`External secrets manager failed to update provider ${provider}`, { + error: ensureError(error), + }); return false; } } diff --git a/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts b/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts index 2c17aee5a6..6c2c0669fb 100644 --- a/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts +++ b/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts @@ -1,8 +1,10 @@ import type { INodeProperties } from 'n8n-workflow'; +import Container from 'typedi'; import { UnknownAuthTypeError } from '@/errors/unknown-auth-type.error'; import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; import type { SecretsProvider, SecretsProviderState } from '@/interfaces'; +import { Logger } from '@/logging/logger.service'; import { AwsSecretsClient } from './aws-secrets-client'; import type { AwsSecretsManagerContext } from './types'; @@ -76,10 +78,16 @@ export class AwsSecretsManager implements SecretsProvider { private client: AwsSecretsClient; + constructor(private readonly logger = Container.get(Logger)) { + this.logger = this.logger.scoped('external-secrets'); + } + async init(context: AwsSecretsManagerContext) { this.assertAuthType(context); this.client = new AwsSecretsClient(context.settings); + + this.logger.debug('AWS Secrets Manager provider initialized'); } async test() { @@ -87,9 +95,15 @@ export class AwsSecretsManager implements SecretsProvider { } async connect() { - const [wasSuccessful] = await this.test(); + const [wasSuccessful, errorMsg] = await this.test(); this.state = wasSuccessful ? 'connected' : 'error'; + + if (wasSuccessful) { + this.logger.debug('AWS Secrets Manager provider connected'); + } else { + this.logger.error('AWS Secrets Manager provider failed to connect', { errorMsg }); + } } async disconnect() { @@ -104,6 +118,8 @@ export class AwsSecretsManager implements SecretsProvider { this.cachedSecrets = Object.fromEntries( supportedSecrets.map((s) => [s.secretName, s.secretValue]), ); + + this.logger.debug('AWS Secrets Manager provider secrets updated'); } getSecret(name: string) { diff --git a/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts b/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts index c87c5e07d0..7961f21bad 100644 --- a/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts +++ b/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts @@ -1,9 +1,11 @@ -import { ClientSecretCredential } from '@azure/identity'; -import { SecretClient } from '@azure/keyvault-secrets'; +import type { SecretClient } from '@azure/keyvault-secrets'; +import { ensureError } from 'n8n-workflow'; import type { INodeProperties } from 'n8n-workflow'; +import Container from 'typedi'; import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; import type { SecretsProvider, SecretsProviderState } from '@/interfaces'; +import { Logger } from '@/logging/logger.service'; import type { AzureKeyVaultContext } from './types'; @@ -65,19 +67,32 @@ export class AzureKeyVault implements SecretsProvider { private settings: AzureKeyVaultContext['settings']; + constructor(private readonly logger = Container.get(Logger)) { + this.logger = this.logger.scoped('external-secrets'); + } + async init(context: AzureKeyVaultContext) { this.settings = context.settings; + + this.logger.debug('Azure Key Vault provider initialized'); } async connect() { const { vaultName, tenantId, clientId, clientSecret } = this.settings; + const { ClientSecretCredential } = await import('@azure/identity'); + const { SecretClient } = await import('@azure/keyvault-secrets'); + try { const credential = new ClientSecretCredential(tenantId, clientId, clientSecret); this.client = new SecretClient(`https://${vaultName}.vault.azure.net/`, credential); this.state = 'connected'; - } catch { + this.logger.debug('Azure Key Vault provider connected'); + } catch (error) { this.state = 'error'; + this.logger.error('Azure Key Vault provider failed to connect', { + error: ensureError(error), + }); } } @@ -117,6 +132,8 @@ export class AzureKeyVault implements SecretsProvider { acc[cur.name] = cur.value; return acc; }, {}); + + this.logger.debug('Azure Key Vault provider secrets updated'); } getSecret(name: string) { diff --git a/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts index c562139105..c4bf71cb72 100644 --- a/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts +++ b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts @@ -1,8 +1,10 @@ -import { SecretManagerServiceClient as GcpClient } from '@google-cloud/secret-manager'; -import { jsonParse, type INodeProperties } from 'n8n-workflow'; +import type { SecretManagerServiceClient as GcpClient } from '@google-cloud/secret-manager'; +import { ensureError, jsonParse, type INodeProperties } from 'n8n-workflow'; +import Container from 'typedi'; import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; import type { SecretsProvider, SecretsProviderState } from '@/interfaces'; +import { Logger } from '@/logging/logger.service'; import type { GcpSecretsManagerContext, @@ -38,6 +40,10 @@ export class GcpSecretsManager implements SecretsProvider { private settings: GcpSecretAccountKey; + constructor(private readonly logger = Container.get(Logger)) { + this.logger = this.logger.scoped('external-secrets'); + } + async init(context: GcpSecretsManagerContext) { this.settings = this.parseSecretAccountKey(context.settings.serviceAccountKey); } @@ -45,14 +51,20 @@ export class GcpSecretsManager implements SecretsProvider { async connect() { const { projectId, privateKey, clientEmail } = this.settings; + const { SecretManagerServiceClient: GcpClient } = await import('@google-cloud/secret-manager'); + try { this.client = new GcpClient({ credentials: { client_email: clientEmail, private_key: privateKey }, projectId, }); this.state = 'connected'; - } catch { + this.logger.debug('GCP Secrets Manager provider connected'); + } catch (error) { this.state = 'error'; + this.logger.debug('GCP Secrets Manager provider failed to connect', { + error: ensureError(error), + }); } } @@ -112,6 +124,8 @@ export class GcpSecretsManager implements SecretsProvider { if (cur) acc[cur.name] = cur.value; return acc; }, {}); + + this.logger.debug('GCP Secrets Manager provider secrets updated'); } getSecret(name: string) { diff --git a/packages/cli/src/external-secrets/providers/vault.ts b/packages/cli/src/external-secrets/providers/vault.ts index 398c40745d..0f1e93a5da 100644 --- a/packages/cli/src/external-secrets/providers/vault.ts +++ b/packages/cli/src/external-secrets/providers/vault.ts @@ -237,6 +237,7 @@ export class VaultProvider extends SecretsProvider { constructor(readonly logger = Container.get(Logger)) { super(); + this.logger = this.logger.scoped('external-secrets'); } async init(settings: SecretsProviderSettings): Promise { @@ -257,6 +258,8 @@ export class VaultProvider extends SecretsProvider { } return config; }); + + this.logger.debug('Vault provider initialized'); } async connect(): Promise { @@ -408,6 +411,7 @@ export class VaultProvider extends SecretsProvider { kvVersion: string, path: string, ): Promise<[string, IDataObject] | null> { + this.logger.debug(`Getting kv secrets from ${mountPath}${path} (version ${kvVersion})`); let listPath = mountPath; if (kvVersion === '2') { listPath += 'metadata/'; @@ -441,6 +445,7 @@ export class VaultProvider extends SecretsProvider { secretPath += path + key; try { const secretResp = await this.#http.get>(secretPath); + this.logger.debug(`Vault provider retrieved secrets from ${secretPath}`); return [ key, kvVersion === '2' @@ -457,6 +462,7 @@ export class VaultProvider extends SecretsProvider { .filter((v): v is [string, IDataObject] => v !== null), ); const name = path.substring(0, path.length - 1); + this.logger.debug(`Vault provider retrieved kv secrets from ${name}`); return [name, data]; } @@ -479,6 +485,7 @@ export class VaultProvider extends SecretsProvider { ).filter((v): v is [string, IDataObject] => v !== null), ); this.cachedSecrets = secrets; + this.logger.debug('Vault provider secrets updated'); } async test(): Promise<[boolean] | [boolean, string]> { diff --git a/packages/cli/src/interfaces.ts b/packages/cli/src/interfaces.ts index 4d2cd9b2d9..65b3a56346 100644 --- a/packages/cli/src/interfaces.ts +++ b/packages/cli/src/interfaces.ts @@ -355,7 +355,7 @@ export type NumericLicenseFeature = ValuesOf; export interface ILicenseReadResponse { usage: { - executions: { + activeWorkflowTriggers: { limit: number; value: number; warningThreshold: number; diff --git a/packages/cli/src/license.ts b/packages/cli/src/license.ts index c6e8dfa19f..8f1bd26e64 100644 --- a/packages/cli/src/license.ts +++ b/packages/cli/src/license.ts @@ -1,3 +1,4 @@ +import { GlobalConfig } from '@n8n/config'; import type { TEntitlement, TFeatures, TLicenseBlock } from '@n8n_io/license-sdk'; import { LicenseManager } from '@n8n_io/license-sdk'; import { InstanceSettings, ObjectStoreService } from 'n8n-core'; @@ -37,8 +38,9 @@ export class License { private readonly orchestrationService: OrchestrationService, private readonly settingsRepository: SettingsRepository, private readonly licenseMetricsService: LicenseMetricsService, + private readonly globalConfig: GlobalConfig, ) { - this.logger = this.logger.withScope('license'); + this.logger = this.logger.scoped('license'); } /** @@ -46,15 +48,14 @@ export class License { */ private renewalEnabled() { if (this.instanceSettings.instanceType !== 'main') return false; - - const autoRenewEnabled = config.getEnv('license.autoRenewEnabled'); + const autoRenewEnabled = this.globalConfig.license.autoRenewalEnabled; /** * In multi-main setup, all mains start off with `unset` status and so renewal disabled. * On becoming leader or follower, each will enable or disable renewal, respectively. * This ensures the mains do not cause a 429 (too many requests) on license init. */ - if (config.getEnv('multiMainSetup.enabled')) { + if (this.globalConfig.multiMainSetup.enabled) { return autoRenewEnabled && this.instanceSettings.isLeader; } @@ -73,9 +74,9 @@ export class License { const { instanceType } = this.instanceSettings; const isMainInstance = instanceType === 'main'; - const server = config.getEnv('license.serverUrl'); + const server = this.globalConfig.license.serverUrl; const offlineMode = !isMainInstance; - const autoRenewOffset = config.getEnv('license.autoRenewOffset'); + const autoRenewOffset = this.globalConfig.license.autoRenewOffset; const saveCertStr = isMainInstance ? async (value: TLicenseBlock) => await this.saveCertStr(value) : async () => {}; @@ -94,7 +95,7 @@ export class License { try { this.manager = new LicenseManager({ server, - tenantId: config.getEnv('license.tenantId'), + tenantId: this.globalConfig.license.tenantId, productIdentifier: `n8n-${N8N_VERSION}`, autoRenewEnabled: renewalEnabled, renewOnInit: renewalEnabled, @@ -120,7 +121,7 @@ export class License { async loadCertStr(): Promise { // if we have an ephemeral license, we don't want to load it from the database - const ephemeralLicense = config.get('license.cert'); + const ephemeralLicense = this.globalConfig.license.cert; if (ephemeralLicense) { return ephemeralLicense; } @@ -136,17 +137,14 @@ export class License { async onFeatureChange(_features: TFeatures): Promise { this.logger.debug('License feature change detected', _features); - if (config.getEnv('executions.mode') === 'queue' && config.getEnv('multiMainSetup.enabled')) { + if (config.getEnv('executions.mode') === 'queue' && this.globalConfig.multiMainSetup.enabled) { const isMultiMainLicensed = _features[LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES] as | boolean | undefined; this.orchestrationService.setMultiMainSetupLicensed(isMultiMainLicensed ?? false); - if ( - this.orchestrationService.isMultiMainSetupEnabled && - this.orchestrationService.isFollower - ) { + if (this.orchestrationService.isMultiMainSetupEnabled && this.instanceSettings.isFollower) { this.logger.debug( '[Multi-main setup] Instance is follower, skipping sending of "reload-license" command...', ); @@ -180,7 +178,7 @@ export class License { async saveCertStr(value: TLicenseBlock): Promise { // if we have an ephemeral license, we don't want to save it to the database - if (config.get('license.cert')) return; + if (this.globalConfig.license.cert) return; await this.settingsRepository.upsert( { key: SETTINGS_LICENSE_CERT_KEY, diff --git a/packages/cli/src/license/__tests__/license.service.test.ts b/packages/cli/src/license/__tests__/license.service.test.ts index 77afe04a2c..8ffc1dbf3d 100644 --- a/packages/cli/src/license/__tests__/license.service.test.ts +++ b/packages/cli/src/license/__tests__/license.service.test.ts @@ -41,7 +41,7 @@ describe('LicenseService', () => { const data = await licenseService.getLicenseData(); expect(data).toEqual({ usage: { - executions: { + activeWorkflowTriggers: { limit: 400, value: 7, warningThreshold: 0.8, @@ -94,6 +94,7 @@ describe('LicenseService', () => { .spyOn(axios, 'post') .mockResolvedValueOnce({ data: { title: 'Title', text: 'Text', licenseKey: 'abc-123' } }); const data = await licenseService.registerCommunityEdition({ + userId: '123', email: 'test@ema.il', instanceId: '123', instanceUrl: 'http://localhost', @@ -102,6 +103,7 @@ describe('LicenseService', () => { expect(data).toEqual({ title: 'Title', text: 'Text' }); expect(eventService.emit).toHaveBeenCalledWith('license-community-plus-registered', { + userId: '123', email: 'test@ema.il', licenseKey: 'abc-123', }); @@ -111,6 +113,7 @@ describe('LicenseService', () => { jest.spyOn(axios, 'post').mockRejectedValueOnce(new AxiosError('Failed')); await expect( licenseService.registerCommunityEdition({ + userId: '123', email: 'test@ema.il', instanceId: '123', instanceUrl: 'http://localhost', diff --git a/packages/cli/src/license/license.controller.ts b/packages/cli/src/license/license.controller.ts index db895ef4a0..3c284a25cb 100644 --- a/packages/cli/src/license/license.controller.ts +++ b/packages/cli/src/license/license.controller.ts @@ -4,7 +4,7 @@ import { InstanceSettings } from 'n8n-core'; import { Get, Post, RestController, GlobalScope, Body } from '@/decorators'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { AuthenticatedRequest, AuthlessRequest, LicenseRequest } from '@/requests'; +import { AuthenticatedRequest, LicenseRequest } from '@/requests'; import { UrlService } from '@/services/url.service'; import { LicenseService } from './license.service'; @@ -41,11 +41,12 @@ export class LicenseController { @Post('/enterprise/community-registered') async registerCommunityEdition( - _req: AuthlessRequest, + req: AuthenticatedRequest, _res: Response, @Body payload: CommunityRegisteredRequestDto, ) { return await this.licenseService.registerCommunityEdition({ + userId: req.user.id, email: payload.email, instanceId: this.instanceSettings.instanceId, instanceUrl: this.urlService.getInstanceBaseUrl(), diff --git a/packages/cli/src/license/license.service.ts b/packages/cli/src/license/license.service.ts index 43f9961334..1419d58b83 100644 --- a/packages/cli/src/license/license.service.ts +++ b/packages/cli/src/license/license.service.ts @@ -37,7 +37,7 @@ export class LicenseService { return { usage: { - executions: { + activeWorkflowTriggers: { value: triggerCount, limit: this.license.getTriggerLimit(), warningThreshold: 0.8, @@ -61,11 +61,13 @@ export class LicenseService { } async registerCommunityEdition({ + userId, email, instanceId, instanceUrl, licenseType, }: { + userId: User['id']; email: string; instanceId: string; instanceUrl: string; @@ -83,7 +85,7 @@ export class LicenseService { licenseType, }, ); - this.eventService.emit('license-community-plus-registered', { email, licenseKey }); + this.eventService.emit('license-community-plus-registered', { userId, email, licenseKey }); return rest; } catch (e: unknown) { if (e instanceof AxiosError) { diff --git a/packages/cli/src/logging/__tests__/logger.service.test.ts b/packages/cli/src/logging/__tests__/logger.service.test.ts index d01a709639..2ffbf2120e 100644 --- a/packages/cli/src/logging/__tests__/logger.service.test.ts +++ b/packages/cli/src/logging/__tests__/logger.service.test.ts @@ -1,10 +1,42 @@ +jest.mock('n8n-workflow', () => ({ + ...jest.requireActual('n8n-workflow'), + LoggerProxy: { init: jest.fn() }, +})); + import type { GlobalConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; import type { InstanceSettings } from 'n8n-core'; +import { LoggerProxy } from 'n8n-workflow'; import { Logger } from '@/logging/logger.service'; describe('Logger', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('constructor', () => { + const globalConfig = mock({ + logging: { + level: 'info', + outputs: ['console'], + scopes: [], + }, + }); + + test('if root, should initialize `LoggerProxy` with instance', () => { + const logger = new Logger(globalConfig, mock(), { isRoot: true }); + + expect(LoggerProxy.init).toHaveBeenCalledWith(logger); + }); + + test('if scoped, should not initialize `LoggerProxy`', () => { + new Logger(globalConfig, mock(), { isRoot: false }); + + expect(LoggerProxy.init).not.toHaveBeenCalled(); + }); + }); + describe('transports', () => { test('if `console` selected, should set console transport', () => { const globalConfig = mock({ diff --git a/packages/cli/src/logging/logger.service.ts b/packages/cli/src/logging/logger.service.ts index 8bdb9177de..46471c0611 100644 --- a/packages/cli/src/logging/logger.service.ts +++ b/packages/cli/src/logging/logger.service.ts @@ -30,6 +30,7 @@ export class Logger { constructor( private readonly globalConfig: GlobalConfig, private readonly instanceSettings: InstanceSettings, + { isRoot }: { isRoot?: boolean } = { isRoot: true }, ) { this.level = this.globalConfig.logging.level; @@ -51,16 +52,18 @@ export class Logger { this.scopes = new Set(scopes); } - LoggerProxy.init(this); + if (isRoot) LoggerProxy.init(this); } private setInternalLogger(internalLogger: winston.Logger) { this.internalLogger = internalLogger; } - withScope(scope: LogScope) { - const scopedLogger = new Logger(this.globalConfig, this.instanceSettings); - const childLogger = this.internalLogger.child({ scope }); + /** Create a logger that injects the given scopes into its log metadata. */ + scoped(scopes: LogScope | LogScope[]) { + scopes = Array.isArray(scopes) ? scopes : [scopes]; + const scopedLogger = new Logger(this.globalConfig, this.instanceSettings, { isRoot: false }); + const childLogger = this.internalLogger.child({ scopes }); scopedLogger.setInternalLogger(childLogger); @@ -106,11 +109,14 @@ export class Logger { private scopeFilter() { return winston.format((info: TransformableInfo & { metadata: LogMetadata }) => { - const shouldIncludeScope = info.metadata.scope && this.scopes.has(info.metadata.scope); + if (!this.isScopingEnabled) return info; - if (this.isScopingEnabled && !shouldIncludeScope) return false; + const { scopes } = info.metadata; - return info; + const shouldIncludeScope = + scopes && scopes?.length > 0 && scopes.some((s) => this.scopes.has(s)); + + return shouldIncludeScope ? info : false; })(); } diff --git a/packages/cli/src/logging/types.ts b/packages/cli/src/logging/types.ts index b6022c0bf6..bb01834326 100644 --- a/packages/cli/src/logging/types.ts +++ b/packages/cli/src/logging/types.ts @@ -6,7 +6,7 @@ export type LogLevel = (typeof LOG_LEVELS)[number]; export type LogMetadata = { [key: string]: unknown; - scope?: LogScope; + scopes?: LogScope[]; file?: string; function?: string; }; diff --git a/packages/cli/src/metrics/__tests__/license-metrics.service.test.ts b/packages/cli/src/metrics/__tests__/license-metrics.service.test.ts index ec123a3f01..6771b13fb7 100644 --- a/packages/cli/src/metrics/__tests__/license-metrics.service.test.ts +++ b/packages/cli/src/metrics/__tests__/license-metrics.service.test.ts @@ -6,11 +6,14 @@ import { LicenseMetricsService } from '@/metrics/license-metrics.service'; describe('LicenseMetricsService', () => { const workflowRepository = mock(); + const licenseMetricsRespository = mock(); const licenseMetricsService = new LicenseMetricsService( - mock(), + licenseMetricsRespository, workflowRepository, ); + beforeEach(() => jest.clearAllMocks()); + describe('collectPassthroughData', () => { test('should return an object with active workflow IDs', async () => { /** @@ -30,4 +33,36 @@ describe('LicenseMetricsService', () => { expect(result).toEqual({ activeWorkflowIds }); }); }); + + describe('collectUsageMetrics', () => { + test('should return an array of expected usage metrics', async () => { + const mockActiveTriggerCount = 1234; + workflowRepository.getActiveTriggerCount.mockResolvedValue(mockActiveTriggerCount); + + const mockRenewalMetrics = { + activeWorkflows: 100, + totalWorkflows: 200, + enabledUsers: 300, + totalUsers: 400, + totalCredentials: 500, + productionExecutions: 600, + manualExecutions: 700, + }; + + licenseMetricsRespository.getLicenseRenewalMetrics.mockResolvedValue(mockRenewalMetrics); + + const result = await licenseMetricsService.collectUsageMetrics(); + + expect(result).toEqual([ + { name: 'activeWorkflows', value: mockRenewalMetrics.activeWorkflows }, + { name: 'totalWorkflows', value: mockRenewalMetrics.totalWorkflows }, + { name: 'enabledUsers', value: mockRenewalMetrics.enabledUsers }, + { name: 'totalUsers', value: mockRenewalMetrics.totalUsers }, + { name: 'totalCredentials', value: mockRenewalMetrics.totalCredentials }, + { name: 'productionExecutions', value: mockRenewalMetrics.productionExecutions }, + { name: 'manualExecutions', value: mockRenewalMetrics.manualExecutions }, + { name: 'activeWorkflowTriggers', value: mockActiveTriggerCount }, + ]); + }); + }); }); diff --git a/packages/cli/src/metrics/__tests__/prometheus-metrics.service.unmocked.test.ts b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.unmocked.test.ts new file mode 100644 index 0000000000..e485bbe435 --- /dev/null +++ b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.unmocked.test.ts @@ -0,0 +1,109 @@ +import { GlobalConfig } from '@n8n/config'; +import type express from 'express'; +import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; +import promClient from 'prom-client'; + +import { EventMessageWorkflow } from '@/eventbus/event-message-classes/event-message-workflow'; +import type { EventService } from '@/events/event.service'; +import { mockInstance } from '@test/mocking'; + +import { MessageEventBus } from '../../eventbus/message-event-bus/message-event-bus'; +import { PrometheusMetricsService } from '../prometheus-metrics.service'; + +jest.unmock('@/eventbus/message-event-bus/message-event-bus'); + +const customPrefix = 'custom_'; + +const eventService = mock(); +const instanceSettings = mock({ instanceType: 'main' }); +const app = mock(); +const eventBus = new MessageEventBus( + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), +); + +describe('workflow_success_total', () => { + test('support workflow id labels', async () => { + // ARRANGE + const globalConfig = mockInstance(GlobalConfig, { + endpoints: { + metrics: { + prefix: '', + includeMessageEventBusMetrics: true, + includeWorkflowIdLabel: true, + }, + }, + }); + + const prometheusMetricsService = new PrometheusMetricsService( + mock(), + eventBus, + globalConfig, + eventService, + instanceSettings, + ); + + await prometheusMetricsService.init(app); + + // ACT + const event = new EventMessageWorkflow({ + eventName: 'n8n.workflow.success', + payload: { workflowId: '1234' }, + }); + + eventBus.emit('metrics.eventBus.event', event); + + // ASSERT + const workflowSuccessCounter = + await promClient.register.getSingleMetricAsString('workflow_success_total'); + + expect(workflowSuccessCounter).toMatchInlineSnapshot(` +"# HELP workflow_success_total Total number of n8n.workflow.success events. +# TYPE workflow_success_total counter +workflow_success_total{workflow_id="1234"} 1" +`); + }); + + test('support a custom prefix', async () => { + // ARRANGE + const globalConfig = mockInstance(GlobalConfig, { + endpoints: { + metrics: { + prefix: customPrefix, + }, + }, + }); + + const prometheusMetricsService = new PrometheusMetricsService( + mock(), + eventBus, + globalConfig, + eventService, + instanceSettings, + ); + + await prometheusMetricsService.init(app); + + // ACT + const event = new EventMessageWorkflow({ + eventName: 'n8n.workflow.success', + payload: { workflowId: '1234' }, + }); + + eventBus.emit('metrics.eventBus.event', event); + + // ASSERT + const versionInfoMetric = promClient.register.getSingleMetric(`${customPrefix}version_info`); + + if (!versionInfoMetric) { + fail(`Could not find a metric called "${customPrefix}version_info"`); + } + }); +}); diff --git a/packages/cli/src/metrics/license-metrics.service.ts b/packages/cli/src/metrics/license-metrics.service.ts index ba546b3cc0..338fa70a3e 100644 --- a/packages/cli/src/metrics/license-metrics.service.ts +++ b/packages/cli/src/metrics/license-metrics.service.ts @@ -21,6 +21,8 @@ export class LicenseMetricsService { manualExecutions, } = await this.licenseMetricsRepository.getLicenseRenewalMetrics(); + const activeTriggerCount = await this.workflowRepository.getActiveTriggerCount(); + return [ { name: 'activeWorkflows', value: activeWorkflows }, { name: 'totalWorkflows', value: totalWorkflows }, @@ -29,6 +31,7 @@ export class LicenseMetricsService { { name: 'totalCredentials', value: totalCredentials }, { name: 'productionExecutions', value: productionExecutions }, { name: 'manualExecutions', value: manualExecutions }, + { name: 'activeWorkflowTriggers', value: activeTriggerCount }, ]; } diff --git a/packages/cli/src/metrics/prometheus-metrics.service.ts b/packages/cli/src/metrics/prometheus-metrics.service.ts index 2565b0a6b1..41714d25ad 100644 --- a/packages/cli/src/metrics/prometheus-metrics.service.ts +++ b/packages/cli/src/metrics/prometheus-metrics.service.ts @@ -211,7 +211,6 @@ export class PrometheusMetricsService { help: `Total number of ${eventName} events.`, labelNames: Object.keys(labels), }); - counter.labels(labels).inc(0); this.counters[eventName] = counter; } @@ -224,7 +223,9 @@ export class PrometheusMetricsService { this.eventBus.on('metrics.eventBus.event', (event: EventMessageTypes) => { const counter = this.toCounter(event); if (!counter) return; - counter.inc(1); + + const labels = this.toLabels(event); + counter.inc(labels, 1); }); } diff --git a/packages/cli/src/node-types.ts b/packages/cli/src/node-types.ts index 84b406001e..26b1b61e36 100644 --- a/packages/cli/src/node-types.ts +++ b/packages/cli/src/node-types.ts @@ -44,15 +44,38 @@ export class NodeTypes implements INodeTypes { } getByNameAndVersion(nodeType: string, version?: number): INodeType { - const versionedNodeType = NodeHelpers.getVersionedNodeType( - this.getNode(nodeType).type, - version, - ); - if (versionedNodeType.description.usableAsTool) { - return NodeHelpers.convertNodeToAiTool(versionedNodeType); + const origType = nodeType; + const toolRequested = nodeType.startsWith('n8n-nodes-base') && nodeType.endsWith('Tool'); + // Make sure the nodeType to actually get from disk is the un-wrapped type + if (toolRequested) { + nodeType = nodeType.replace(/Tool$/, ''); } - return versionedNodeType; + const node = this.getNode(nodeType); + const versionedNodeType = NodeHelpers.getVersionedNodeType(node.type, version); + if (!toolRequested) return versionedNodeType; + + if (!versionedNodeType.description.usableAsTool) + throw new ApplicationError('Node cannot be used as a tool', { extra: { nodeType } }); + + const { loadedNodes } = this.loadNodesAndCredentials; + if (origType in loadedNodes) { + return loadedNodes[origType].type as INodeType; + } + + // Instead of modifying the existing type, we extend it into a new type object + const clonedProperties = Object.create( + versionedNodeType.description.properties, + ) as INodeTypeDescription['properties']; + const clonedDescription = Object.create(versionedNodeType.description, { + properties: { value: clonedProperties }, + }) as INodeTypeDescription; + const clonedNode = Object.create(versionedNodeType, { + description: { value: clonedDescription }, + }) as INodeType; + const tool = NodeHelpers.convertNodeToAiTool(clonedNode); + loadedNodes[nodeType + 'Tool'] = { sourcePath: '', type: tool }; + return tool; } /* Some nodeTypes need to get special parameters applied like the polling nodes the polling times */ diff --git a/packages/cli/src/permissions/global-roles.ts b/packages/cli/src/permissions/global-roles.ts index 664cd8384e..7ea1b575da 100644 --- a/packages/cli/src/permissions/global-roles.ts +++ b/packages/cli/src/permissions/global-roles.ts @@ -15,6 +15,7 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [ 'credential:list', 'credential:share', 'credential:move', + 'community:register', 'communityPackage:install', 'communityPackage:uninstall', 'communityPackage:update', @@ -38,7 +39,6 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [ 'license:manage', 'logStreaming:manage', 'orchestration:read', - 'orchestration:list', 'saml:manage', 'securityAudit:generate', 'sourceControl:pull', diff --git a/packages/cli/src/public-api/index.ts b/packages/cli/src/public-api/index.ts index 1264f57496..92b3602828 100644 --- a/packages/cli/src/public-api/index.ts +++ b/packages/cli/src/public-api/index.ts @@ -3,16 +3,13 @@ import type { Router } from 'express'; import express from 'express'; import type { HttpError } from 'express-openapi-validator/dist/framework/types'; import fs from 'fs/promises'; -import type { OpenAPIV3 } from 'openapi-types'; import path from 'path'; import type { JsonObject } from 'swagger-ui-express'; import { Container } from 'typedi'; import validator from 'validator'; import YAML from 'yamljs'; -import { EventService } from '@/events/event.service'; import { License } from '@/license'; -import type { AuthenticatedRequest } from '@/requests'; import { PublicApiKeyService } from '@/services/public-api-key.service'; import { UrlService } from '@/services/url.service'; @@ -85,28 +82,7 @@ async function createApiRouter( }, validateSecurity: { handlers: { - ApiKeyAuth: async ( - req: AuthenticatedRequest, - _scopes: unknown, - schema: OpenAPIV3.ApiKeySecurityScheme, - ): Promise => { - const providedApiKey = req.headers[schema.name.toLowerCase()] as string; - - const user = await Container.get(PublicApiKeyService).getUserForApiKey(providedApiKey); - - if (!user) return false; - - Container.get(EventService).emit('public-api-invoked', { - userId: user.id, - path: req.path, - method: req.method, - apiVersion: version, - }); - - req.user = user; - - return true; - }, + ApiKeyAuth: Container.get(PublicApiKeyService).getAuthMiddleware(version), }, }, }), diff --git a/packages/cli/src/public-api/types.ts b/packages/cli/src/public-api/types.ts index e2d22eac2c..327d363073 100644 --- a/packages/cli/src/public-api/types.ts +++ b/packages/cli/src/public-api/types.ts @@ -140,11 +140,7 @@ export declare namespace CredentialRequest { type Delete = AuthenticatedRequest<{ id: string }, {}, {}, Record>; - type Transfer = AuthenticatedRequest< - { workflowId: string }, - {}, - { destinationProjectId: string } - >; + type Transfer = AuthenticatedRequest<{ id: string }, {}, { destinationProjectId: string }>; } export type OperationID = 'getUsers' | 'getUser'; diff --git a/packages/cli/src/public-api/v1/handlers/credentials/credentials.handler.ts b/packages/cli/src/public-api/v1/handlers/credentials/credentials.handler.ts index 2a0cbe3895..d987aa827e 100644 --- a/packages/cli/src/public-api/v1/handlers/credentials/credentials.handler.ts +++ b/packages/cli/src/public-api/v1/handlers/credentials/credentials.handler.ts @@ -53,7 +53,7 @@ export = { await Container.get(EnterpriseCredentialsService).transferOne( req.user, - req.params.workflowId, + req.params.id, body.destinationProjectId, ); diff --git a/packages/cli/src/public-api/v1/handlers/credentials/spec/paths/credentials.yml b/packages/cli/src/public-api/v1/handlers/credentials/spec/paths/credentials.yml index eab345f572..13a1be6de4 100644 --- a/packages/cli/src/public-api/v1/handlers/credentials/spec/paths/credentials.yml +++ b/packages/cli/src/public-api/v1/handlers/credentials/spec/paths/credentials.yml @@ -18,7 +18,7 @@ post: content: application/json: schema: - $ref: '../schemas/credential.yml' + $ref: '../schemas/create-credential-response.yml' '401': $ref: '../../../../shared/spec/responses/unauthorized.yml' '415': diff --git a/packages/cli/src/public-api/v1/handlers/credentials/spec/schemas/create-credential-response.yml b/packages/cli/src/public-api/v1/handlers/credentials/spec/schemas/create-credential-response.yml new file mode 100644 index 0000000000..fa906cfd29 --- /dev/null +++ b/packages/cli/src/public-api/v1/handlers/credentials/spec/schemas/create-credential-response.yml @@ -0,0 +1,28 @@ +required: + - id + - name + - type + - createdAt + - updatedAt +type: object +properties: + id: + type: string + readOnly: true + example: vHxaz5UaCghVYl9C + name: + type: string + example: John's Github account + type: + type: string + example: github + createdAt: + type: string + format: date-time + readOnly: true + example: '2022-04-29T11:02:29.842Z' + updatedAt: + type: string + format: date-time + readOnly: true + example: '2022-04-29T11:02:29.842Z' diff --git a/packages/cli/src/push/__tests__/index.test.ts b/packages/cli/src/push/__tests__/index.test.ts index 6230c63397..03457926b1 100644 --- a/packages/cli/src/push/__tests__/index.test.ts +++ b/packages/cli/src/push/__tests__/index.test.ts @@ -20,7 +20,7 @@ describe('Push', () => { test('should validate pushRef on requests for websocket backend', () => { config.set('push.backend', 'websocket'); - const push = new Push(mock()); + const push = new Push(mock(), mock()); const ws = mock(); const request = mock({ user, ws }); request.query = { pushRef: '' }; @@ -33,7 +33,7 @@ describe('Push', () => { test('should validate pushRef on requests for SSE backend', () => { config.set('push.backend', 'sse'); - const push = new Push(mock()); + const push = new Push(mock(), mock()); const request = mock({ user, ws: undefined }); request.query = { pushRef: '' }; expect(() => push.handleRequest(request, mock())).toThrow(BadRequestError); diff --git a/packages/cli/src/push/index.ts b/packages/cli/src/push/index.ts index 232864968d..bfbfb43a51 100644 --- a/packages/cli/src/push/index.ts +++ b/packages/cli/src/push/index.ts @@ -12,6 +12,7 @@ import config from '@/config'; import type { User } from '@/databases/entities/user'; import { OnShutdown } from '@/decorators/on-shutdown'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { OrchestrationService } from '@/services/orchestration.service'; import { TypedEmitter } from '@/typed-emitter'; @@ -39,7 +40,10 @@ export class Push extends TypedEmitter { private backend = useWebSockets ? Container.get(WebSocketPush) : Container.get(SSEPush); - constructor(private readonly orchestrationService: OrchestrationService) { + constructor( + private readonly orchestrationService: OrchestrationService, + private readonly publisher: Publisher, + ) { super(); if (useWebSockets) this.backend.on('message', (msg) => this.emit('message', msg)); @@ -89,8 +93,10 @@ export class Push extends TypedEmitter { * relay the former's execution lifecycle events to the creator's frontend. */ if (this.orchestrationService.isMultiMainSetupEnabled && !this.backend.hasPushRef(pushRef)) { - const payload = { type, args: data, pushRef }; - void this.orchestrationService.publish('relay-execution-lifecycle-event', payload); + void this.publisher.publishCommand({ + command: 'relay-execution-lifecycle-event', + payload: { type, args: data, pushRef }, + }); return; } diff --git a/packages/cli/src/requests.ts b/packages/cli/src/requests.ts index e25a244f5f..4765ac1fad 100644 --- a/packages/cli/src/requests.ts +++ b/packages/cli/src/requests.ts @@ -244,7 +244,13 @@ export declare namespace UserRequest { >; export type InviteResponse = { - user: { id: string; email: string; inviteAcceptUrl?: string; emailSent: boolean }; + user: { + id: string; + email: string; + inviteAcceptUrl?: string; + emailSent: boolean; + role: AssignableRole; + }; error?: string; }; @@ -478,15 +484,6 @@ export declare namespace ExternalSecretsRequest { type UpdateProvider = AuthenticatedRequest<{ provider: string }>; } -// ---------------------------------- -// /orchestration -// ---------------------------------- -// -export declare namespace OrchestrationRequest { - type GetAll = AuthenticatedRequest; - type Get = AuthenticatedRequest<{ id: string }, {}, {}, {}>; -} - // ---------------------------------- // /workflow-history // ---------------------------------- diff --git a/packages/cli/src/runners/__tests__/forward-to-logger.test.ts b/packages/cli/src/runners/__tests__/forward-to-logger.test.ts new file mode 100644 index 0000000000..64352ab54d --- /dev/null +++ b/packages/cli/src/runners/__tests__/forward-to-logger.test.ts @@ -0,0 +1,114 @@ +import type { Logger } from 'n8n-workflow'; +import { Readable } from 'stream'; + +import { forwardToLogger } from '../forward-to-logger'; + +describe('forwardToLogger', () => { + let logger: Logger; + let stdout: Readable; + let stderr: Readable; + + beforeEach(() => { + logger = { + info: jest.fn(), + error: jest.fn(), + } as unknown as Logger; + + stdout = new Readable({ read() {} }); + stderr = new Readable({ read() {} }); + + jest.resetAllMocks(); + }); + + const pushToStdout = async (data: string) => { + stdout.push(Buffer.from(data)); + stdout.push(null); + // Wait for the next tick to allow the event loop to process the data + await new Promise((resolve) => setImmediate(resolve)); + }; + + const pushToStderr = async (data: string) => { + stderr.push(Buffer.from(data)); + stderr.push(null); + // Wait for the next tick to allow the event loop to process the data + await new Promise((resolve) => setImmediate(resolve)); + }; + + it('should forward stdout data to logger.info', async () => { + forwardToLogger(logger, { stdout, stderr: null }); + + await pushToStdout('Test stdout message'); + + await new Promise((resolve) => setImmediate(resolve)); + + expect(logger.info).toHaveBeenCalledWith('Test stdout message'); + }); + + it('should forward stderr data to logger.error', async () => { + forwardToLogger(logger, { stdout: null, stderr }); + + await pushToStderr('Test stderr message'); + + expect(logger.error).toHaveBeenCalledWith('Test stderr message'); + }); + + it('should remove trailing newline from stdout', async () => { + forwardToLogger(logger, { stdout, stderr: null }); + + await pushToStdout('Test stdout message\n'); + + expect(logger.info).toHaveBeenCalledWith('Test stdout message'); + }); + + it('should remove trailing newline from stderr', async () => { + forwardToLogger(logger, { stdout: null, stderr }); + + await pushToStderr('Test stderr message\n'); + + expect(logger.error).toHaveBeenCalledWith('Test stderr message'); + }); + + it('should forward stderr data to logger.error', async () => { + forwardToLogger(logger, { stdout: null, stderr }); + + await pushToStderr('Test stderr message'); + + expect(logger.error).toHaveBeenCalledWith('Test stderr message'); + }); + + it('should include prefix if provided for stdout', async () => { + const prefix = '[PREFIX]'; + forwardToLogger(logger, { stdout, stderr: null }, prefix); + + await pushToStdout('Message with prefix'); + + expect(logger.info).toHaveBeenCalledWith('[PREFIX] Message with prefix'); + }); + + it('should include prefix if provided for stderr', async () => { + const prefix = '[PREFIX]'; + forwardToLogger(logger, { stdout: null, stderr }, prefix); + + await pushToStderr('Error message with prefix'); + + expect(logger.error).toHaveBeenCalledWith('[PREFIX] Error message with prefix'); + }); + + it('should make sure there is no duplicate space after prefix for stdout', async () => { + const prefix = '[PREFIX] '; + forwardToLogger(logger, { stdout, stderr: null }, prefix); + + await pushToStdout('Message with prefix'); + + expect(logger.info).toHaveBeenCalledWith('[PREFIX] Message with prefix'); + }); + + it('should make sure there is no duplicate space after prefix for stderr', async () => { + const prefix = '[PREFIX] '; + forwardToLogger(logger, { stdout: null, stderr }, prefix); + + await pushToStderr('Error message with prefix'); + + expect(logger.error).toHaveBeenCalledWith('[PREFIX] Error message with prefix'); + }); +}); diff --git a/packages/cli/src/runners/__tests__/node-process-oom-detector.test.ts b/packages/cli/src/runners/__tests__/node-process-oom-detector.test.ts new file mode 100644 index 0000000000..5b619e0e08 --- /dev/null +++ b/packages/cli/src/runners/__tests__/node-process-oom-detector.test.ts @@ -0,0 +1,43 @@ +import { spawn } from 'node:child_process'; + +import { NodeProcessOomDetector } from '../node-process-oom-detector'; + +describe('NodeProcessOomDetector', () => { + test('should detect an out-of-memory error in a monitored process', (done) => { + const childProcess = spawn(process.execPath, [ + // set low memory limit + '--max-old-space-size=20', + '-e', + ` + const data = []; + // fill memory until it crashes + while (true) data.push(Array.from({ length: 10_000 }).map(() => Math.random().toString()).join()); + `, + ]); + + const detector = new NodeProcessOomDetector(childProcess); + + childProcess.on('exit', (code) => { + expect(detector.didProcessOom).toBe(true); + expect(code).not.toBe(0); + done(); + }); + }); + + test('should not detect an out-of-memory error in a process that exits normally', (done) => { + const childProcess = spawn(process.execPath, [ + '-e', + ` + console.log("Hello, World!"); + `, + ]); + + const detector = new NodeProcessOomDetector(childProcess); + + childProcess.on('exit', (code) => { + expect(detector.didProcessOom).toBe(false); + expect(code).toBe(0); + done(); + }); + }); +}); diff --git a/packages/cli/src/runners/__tests__/sliding-window-signal.test.ts b/packages/cli/src/runners/__tests__/sliding-window-signal.test.ts new file mode 100644 index 0000000000..56462d186a --- /dev/null +++ b/packages/cli/src/runners/__tests__/sliding-window-signal.test.ts @@ -0,0 +1,71 @@ +import { TypedEmitter } from '../../typed-emitter'; +import { SlidingWindowSignal } from '../sliding-window-signal'; + +type TestEventMap = { + testEvent: string; +}; + +describe('SlidingWindowSignal', () => { + let eventEmitter: TypedEmitter; + let slidingWindowSignal: SlidingWindowSignal; + + beforeEach(() => { + eventEmitter = new TypedEmitter(); + slidingWindowSignal = new SlidingWindowSignal(eventEmitter, 'testEvent', { + windowSizeInMs: 500, + }); + }); + + afterEach(() => { + jest.clearAllTimers(); + jest.clearAllMocks(); + }); + + it('should return the last signal if within window size', async () => { + const signal = 'testSignal'; + eventEmitter.emit('testEvent', signal); + + const receivedSignal = await slidingWindowSignal.getSignal(); + + expect(receivedSignal).toBe(signal); + }); + + it('should return null if there is no signal within the window', async () => { + jest.useFakeTimers(); + const receivedSignalPromise = slidingWindowSignal.getSignal(); + jest.advanceTimersByTime(600); + const receivedSignal = await receivedSignalPromise; + + expect(receivedSignal).toBeNull(); + jest.useRealTimers(); + }); + + it('should return null if "exit" event is not emitted before timeout', async () => { + const signal = 'testSignal'; + jest.useFakeTimers(); + const receivedSignalPromise = slidingWindowSignal.getSignal(); + jest.advanceTimersByTime(600); + eventEmitter.emit('testEvent', signal); + + const receivedSignal = await receivedSignalPromise; + expect(receivedSignal).toBeNull(); + jest.useRealTimers(); + }); + + it('should return the signal emitted on "exit" event before timeout', async () => { + jest.useFakeTimers(); + const receivedSignalPromise = slidingWindowSignal.getSignal(); + + // Emit 'exit' with a signal before timeout + const exitSignal = 'exitSignal'; + eventEmitter.emit('testEvent', exitSignal); + + // Advance timers enough to go outside the timeout window + jest.advanceTimersByTime(600); + + const receivedSignal = await receivedSignalPromise; + expect(receivedSignal).toBe(exitSignal); + + jest.useRealTimers(); + }); +}); diff --git a/packages/cli/src/runners/__tests__/task-broker.test.ts b/packages/cli/src/runners/__tests__/task-broker.test.ts index f5b91a3f2c..a90bf7662c 100644 --- a/packages/cli/src/runners/__tests__/task-broker.test.ts +++ b/packages/cli/src/runners/__tests__/task-broker.test.ts @@ -5,24 +5,24 @@ import type { RunnerMessage, TaskResultData } from '../runner-types'; import { TaskBroker } from '../task-broker.service'; import type { TaskOffer, TaskRequest, TaskRunner } from '../task-broker.service'; +const createValidUntil = (ms: number) => process.hrtime.bigint() + BigInt(ms * 1_000_000); + describe('TaskBroker', () => { let taskBroker: TaskBroker; beforeEach(() => { - taskBroker = new TaskBroker(mock()); + taskBroker = new TaskBroker(mock(), mock()); jest.restoreAllMocks(); }); describe('expireTasks', () => { it('should remove expired task offers and keep valid task offers', () => { - const now = process.hrtime.bigint(); - const validOffer: TaskOffer = { offerId: 'valid', runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), // 1 second in the future + validUntil: createValidUntil(1000), // 1 second in the future }; const expiredOffer1: TaskOffer = { @@ -30,7 +30,7 @@ describe('TaskBroker', () => { runnerId: 'runner2', taskType: 'taskType1', validFor: 1000, - validUntil: now - BigInt(1000 * 1_000_000), // 1 second in the past + validUntil: createValidUntil(-1000), // 1 second in the past }; const expiredOffer2: TaskOffer = { @@ -38,7 +38,7 @@ describe('TaskBroker', () => { runnerId: 'runner3', taskType: 'taskType1', validFor: 2000, - validUntil: now - BigInt(2000 * 1_000_000), // 2 seconds in the past + validUntil: createValidUntil(-2000), // 2 seconds in the past }; taskBroker.setPendingTaskOffers([validOffer, expiredOffer1, expiredOffer2]); @@ -69,6 +69,21 @@ describe('TaskBroker', () => { expect(knownRunners.get(runnerId)?.runner).toEqual(runner); expect(knownRunners.get(runnerId)?.messageCallback).toEqual(messageCallback); }); + + it('should send node types to runner', () => { + const runnerId = 'runner1'; + const runner = mock({ id: runnerId }); + const messageCallback = jest.fn(); + + taskBroker.registerRunner(runner, messageCallback); + + expect(messageCallback).toBeCalledWith({ + type: 'broker:nodetypes', + // We're mocking the node types service, so this will + // be undefined. + nodeType: undefined, + }); + }); }); describe('registerRequester', () => { @@ -95,13 +110,66 @@ describe('TaskBroker', () => { const messageCallback = jest.fn(); taskBroker.registerRunner(runner, messageCallback); - taskBroker.deregisterRunner(runnerId); + taskBroker.deregisterRunner(runnerId, new Error()); const knownRunners = taskBroker.getKnownRunners(); const runnerIds = Object.keys(knownRunners); expect(runnerIds).toHaveLength(0); }); + + it('should remove any pending offers for that runner', () => { + const runnerId = 'runner1'; + const runner = mock({ id: runnerId }); + const messageCallback = jest.fn(); + + taskBroker.registerRunner(runner, messageCallback); + taskBroker.taskOffered({ + offerId: 'offer1', + runnerId, + taskType: 'mock', + validFor: 1000, + validUntil: createValidUntil(1000), + }); + taskBroker.taskOffered({ + offerId: 'offer2', + runnerId: 'runner2', + taskType: 'mock', + validFor: 1000, + validUntil: createValidUntil(1000), + }); + taskBroker.deregisterRunner(runnerId, new Error()); + + const offers = taskBroker.getPendingTaskOffers(); + expect(offers).toHaveLength(1); + expect(offers[0].runnerId).toBe('runner2'); + }); + + it('should fail any running tasks for that runner', () => { + const runnerId = 'runner1'; + const runner = mock({ id: runnerId }); + const messageCallback = jest.fn(); + + const taskId = 'task1'; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const failSpy = jest.spyOn(taskBroker as any, 'failTask'); + const rejectSpy = jest.spyOn(taskBroker, 'handleRunnerReject'); + + taskBroker.registerRunner(runner, messageCallback); + taskBroker.setTasks({ + [taskId]: { id: taskId, requesterId: 'requester1', runnerId, taskType: 'mock' }, + task2: { id: 'task2', requesterId: 'requester1', runnerId: 'runner2', taskType: 'mock' }, + }); + const error = new Error('error'); + taskBroker.deregisterRunner(runnerId, error); + + expect(failSpy).toBeCalledWith(taskId, error); + expect(rejectSpy).toBeCalledWith( + taskId, + `The Task Runner (${runnerId}) has disconnected: error`, + ); + }); }); describe('deregisterRequester', () => { @@ -121,14 +189,12 @@ describe('TaskBroker', () => { describe('taskRequested', () => { it('should match a pending offer to an incoming request', async () => { - const now = process.hrtime.bigint(); - const offer: TaskOffer = { offerId: 'offer1', runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), + validUntil: createValidUntil(1000), }; taskBroker.setPendingTaskOffers([offer]); @@ -150,8 +216,6 @@ describe('TaskBroker', () => { describe('taskOffered', () => { it('should match a pending request to an incoming offer', () => { - const now = process.hrtime.bigint(); - const request: TaskRequest = { requestId: 'request1', requesterId: 'requester1', @@ -166,7 +230,7 @@ describe('TaskBroker', () => { runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), + validUntil: createValidUntil(1000), }; jest.spyOn(taskBroker, 'acceptOffer').mockResolvedValue(); // allow Jest to exit cleanly @@ -180,14 +244,12 @@ describe('TaskBroker', () => { describe('settleTasks', () => { it('should match task offers with task requests by task type', () => { - const now = process.hrtime.bigint(); - const offer1: TaskOffer = { offerId: 'offer1', runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), + validUntil: createValidUntil(1000), }; const offer2: TaskOffer = { @@ -195,7 +257,7 @@ describe('TaskBroker', () => { runnerId: 'runner2', taskType: 'taskType2', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), + validUntil: createValidUntil(1000), }; const request1: TaskRequest = { @@ -235,14 +297,12 @@ describe('TaskBroker', () => { }); it('should not match a request whose acceptance is in progress', () => { - const now = process.hrtime.bigint(); - const offer: TaskOffer = { offerId: 'offer1', runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), + validUntil: createValidUntil(1000), }; const request: TaskRequest = { @@ -271,14 +331,12 @@ describe('TaskBroker', () => { }); it('should expire tasks before settling', () => { - const now = process.hrtime.bigint(); - const validOffer: TaskOffer = { offerId: 'valid', runnerId: 'runner1', taskType: 'taskType1', validFor: 1000, - validUntil: now + BigInt(1000 * 1_000_000), // 1 second in the future + validUntil: createValidUntil(1000), // 1 second in the future }; const expiredOffer: TaskOffer = { @@ -286,7 +344,7 @@ describe('TaskBroker', () => { runnerId: 'runner2', taskType: 'taskType2', // will be removed before matching validFor: 1000, - validUntil: now - BigInt(1000 * 1_000_000), // 1 second in the past + validUntil: createValidUntil(-1000), // 1 second in the past }; const request1: TaskRequest = { diff --git a/packages/cli/src/runners/__tests__/task-runner-process.test.ts b/packages/cli/src/runners/__tests__/task-runner-process.test.ts index b2ad678ee1..eb04e3ab8e 100644 --- a/packages/cli/src/runners/__tests__/task-runner-process.test.ts +++ b/packages/cli/src/runners/__tests__/task-runner-process.test.ts @@ -1,10 +1,11 @@ -import { GlobalConfig } from '@n8n/config'; +import { TaskRunnersConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; import type { ChildProcess, SpawnOptions } from 'node:child_process'; -import { mockInstance } from '../../../test/shared/mocking'; -import type { TaskRunnerAuthService } from '../auth/task-runner-auth.service'; -import { TaskRunnerProcess } from '../task-runner-process'; +import { Logger } from '@/logging/logger.service'; +import type { TaskRunnerAuthService } from '@/runners/auth/task-runner-auth.service'; +import { TaskRunnerProcess } from '@/runners/task-runner-process'; +import { mockInstance } from '@test/mocking'; const spawnMock = jest.fn(() => mock({ @@ -19,19 +20,53 @@ const spawnMock = jest.fn(() => require('child_process').spawn = spawnMock; describe('TaskRunnerProcess', () => { - const globalConfig = mockInstance(GlobalConfig); + const logger = mockInstance(Logger); + const runnerConfig = mockInstance(TaskRunnersConfig); + runnerConfig.disabled = false; + runnerConfig.mode = 'internal_childprocess'; const authService = mock(); - const taskRunnerProcess = new TaskRunnerProcess(globalConfig, authService); + let taskRunnerProcess = new TaskRunnerProcess(logger, runnerConfig, authService); afterEach(async () => { spawnMock.mockClear(); }); + describe('constructor', () => { + it('should throw if runner mode is external', () => { + runnerConfig.mode = 'external'; + + expect(() => new TaskRunnerProcess(logger, runnerConfig, authService)).toThrow(); + + runnerConfig.mode = 'internal_childprocess'; + }); + }); + describe('start', () => { - it('should propagate NODE_FUNCTION_ALLOW_BUILTIN and NODE_FUNCTION_ALLOW_EXTERNAL from env', async () => { + beforeEach(() => { + taskRunnerProcess = new TaskRunnerProcess(logger, runnerConfig, authService); + }); + + test.each(['PATH', 'NODE_FUNCTION_ALLOW_BUILTIN', 'NODE_FUNCTION_ALLOW_EXTERNAL'])( + 'should propagate %s from env as is', + async (envVar) => { + jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); + process.env[envVar] = 'custom value'; + + await taskRunnerProcess.start(); + + // @ts-expect-error The type is not correct + const options = spawnMock.mock.calls[0][2] as SpawnOptions; + expect(options.env).toEqual( + expect.objectContaining({ + [envVar]: 'custom value', + }), + ); + }, + ); + + it('should pass NODE_OPTIONS env if maxOldSpaceSize is configured', async () => { jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); - process.env.NODE_FUNCTION_ALLOW_BUILTIN = '*'; - process.env.NODE_FUNCTION_ALLOW_EXTERNAL = '*'; + runnerConfig.maxOldSpaceSize = '1024'; await taskRunnerProcess.start(); @@ -39,10 +74,20 @@ describe('TaskRunnerProcess', () => { const options = spawnMock.mock.calls[0][2] as SpawnOptions; expect(options.env).toEqual( expect.objectContaining({ - NODE_FUNCTION_ALLOW_BUILTIN: '*', - NODE_FUNCTION_ALLOW_EXTERNAL: '*', + NODE_OPTIONS: '--max-old-space-size=1024', }), ); }); + + it('should not pass NODE_OPTIONS env if maxOldSpaceSize is not configured', async () => { + jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); + runnerConfig.maxOldSpaceSize = ''; + + await taskRunnerProcess.start(); + + // @ts-expect-error The type is not correct + const options = spawnMock.mock.calls[0][2] as SpawnOptions; + expect(options.env).not.toHaveProperty('NODE_OPTIONS'); + }); }); }); diff --git a/packages/cli/src/runners/errors/task-runner-disconnected-error.ts b/packages/cli/src/runners/errors/task-runner-disconnected-error.ts new file mode 100644 index 0000000000..6c7c49450a --- /dev/null +++ b/packages/cli/src/runners/errors/task-runner-disconnected-error.ts @@ -0,0 +1,7 @@ +import { ApplicationError } from 'n8n-workflow'; + +export class TaskRunnerDisconnectedError extends ApplicationError { + constructor(runnerId: string) { + super(`Task runner (${runnerId}) disconnected`); + } +} diff --git a/packages/cli/src/runners/errors/task-runner-oom-error.ts b/packages/cli/src/runners/errors/task-runner-oom-error.ts new file mode 100644 index 0000000000..e52b8b4bea --- /dev/null +++ b/packages/cli/src/runners/errors/task-runner-oom-error.ts @@ -0,0 +1,31 @@ +import { ApplicationError } from 'n8n-workflow'; + +import type { TaskRunner } from '../task-broker.service'; + +export class TaskRunnerOomError extends ApplicationError { + public description: string; + + constructor(runnerId: TaskRunner['id'], isCloudDeployment: boolean) { + super(`Task runner (${runnerId}) ran out of memory.`, { level: 'error' }); + + const fixSuggestions = { + reduceItems: 'Reduce the number of items processed at a time by batching the input.', + increaseMemory: + "Increase the memory available to the task runner with 'N8N_RUNNERS_MAX_OLD_SPACE_SIZE' environment variable.", + upgradePlan: 'Upgrade your cloud plan to increase the available memory.', + }; + + const subtitle = + 'The runner executing the code ran out of memory. This usually happens when there are too many items to process. You can try the following:'; + const suggestions = isCloudDeployment + ? [fixSuggestions.reduceItems, fixSuggestions.upgradePlan] + : [fixSuggestions.reduceItems, fixSuggestions.increaseMemory]; + const suggestionsText = suggestions + .map((suggestion, index) => `${index + 1}. ${suggestion}`) + .join('
'); + + const description = `${subtitle}

${suggestionsText}`; + + this.description = description; + } +} diff --git a/packages/cli/src/runners/forward-to-logger.ts b/packages/cli/src/runners/forward-to-logger.ts new file mode 100644 index 0000000000..0bcc813225 --- /dev/null +++ b/packages/cli/src/runners/forward-to-logger.ts @@ -0,0 +1,42 @@ +import type { Logger } from 'n8n-workflow'; +import type { Readable } from 'stream'; + +/** + * Forwards stdout and stderr of a given producer to the given + * logger's info and error methods respectively. + */ +export function forwardToLogger( + logger: Logger, + producer: { + stdout?: Readable | null; + stderr?: Readable | null; + }, + prefix?: string, +) { + if (prefix) { + prefix = prefix.trimEnd(); + } + + const stringify = (data: Buffer) => { + let str = data.toString(); + + // Remove possible trailing newline (otherwise it's duplicated) + if (str.endsWith('\n')) { + str = str.slice(0, -1); + } + + return prefix ? `${prefix} ${str}` : str; + }; + + if (producer.stdout) { + producer.stdout.on('data', (data: Buffer) => { + logger.info(stringify(data)); + }); + } + + if (producer.stderr) { + producer.stderr.on('data', (data: Buffer) => { + logger.error(stringify(data)); + }); + } +} diff --git a/packages/cli/src/runners/node-process-oom-detector.ts b/packages/cli/src/runners/node-process-oom-detector.ts new file mode 100644 index 0000000000..e6debb8551 --- /dev/null +++ b/packages/cli/src/runners/node-process-oom-detector.ts @@ -0,0 +1,34 @@ +import * as a from 'node:assert/strict'; +import type { ChildProcess } from 'node:child_process'; + +/** + * Class to monitor a nodejs process and detect if it runs out of + * memory (OOMs). + */ +export class NodeProcessOomDetector { + public get didProcessOom() { + return this._didProcessOom; + } + + private _didProcessOom = false; + + constructor(processToMonitor: ChildProcess) { + this.monitorProcess(processToMonitor); + } + + private monitorProcess(processToMonitor: ChildProcess) { + a.ok(processToMonitor.stderr, "Can't monitor a process without stderr"); + + processToMonitor.stderr.on('data', this.onStderr); + + processToMonitor.once('exit', () => { + processToMonitor.stderr?.off('data', this.onStderr); + }); + } + + private onStderr = (data: Buffer) => { + if (data.includes('JavaScript heap out of memory')) { + this._didProcessOom = true; + } + }; +} diff --git a/packages/cli/src/runners/runner-types.ts b/packages/cli/src/runners/runner-types.ts index f615754e02..8b205de801 100644 --- a/packages/cli/src/runners/runner-types.ts +++ b/packages/cli/src/runners/runner-types.ts @@ -1,5 +1,5 @@ import type { Response } from 'express'; -import type { INodeExecutionData } from 'n8n-workflow'; +import type { INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow'; import type WebSocket from 'ws'; import type { TaskRunner } from './task-broker.service'; @@ -62,6 +62,11 @@ export namespace N8nMessage { data: unknown; } + export interface NodeTypes { + type: 'broker:nodetypes'; + nodeTypes: INodeTypeBaseDescription[]; + } + export type All = | InfoRequest | TaskOfferAccept @@ -69,7 +74,8 @@ export namespace N8nMessage { | TaskSettings | RunnerRegistered | RPCResponse - | TaskDataResponse; + | TaskDataResponse + | NodeTypes; } export namespace ToRequester { diff --git a/packages/cli/src/runners/runner-ws-server.ts b/packages/cli/src/runners/runner-ws-server.ts index ef9e52f5f5..59bb92ff76 100644 --- a/packages/cli/src/runners/runner-ws-server.ts +++ b/packages/cli/src/runners/runner-ws-server.ts @@ -1,16 +1,7 @@ -import { GlobalConfig } from '@n8n/config'; -import type { Application } from 'express'; -import { ServerResponse, type Server } from 'http'; -import { ApplicationError } from 'n8n-workflow'; -import type { Socket } from 'net'; -import Container, { Service } from 'typedi'; -import { parse as parseUrl } from 'url'; -import { Server as WSServer } from 'ws'; +import { Service } from 'typedi'; import type WebSocket from 'ws'; import { Logger } from '@/logging/logger.service'; -import { send } from '@/response-helper'; -import { TaskRunnerAuthController } from '@/runners/auth/task-runner-auth.controller'; import type { RunnerMessage, @@ -19,29 +10,12 @@ import type { TaskRunnerServerInitResponse, } from './runner-types'; import { TaskBroker, type MessageCallback, type TaskRunner } from './task-broker.service'; +import { TaskRunnerDisconnectAnalyzer } from './task-runner-disconnect-analyzer'; function heartbeat(this: WebSocket) { this.isAlive = true; } -function getEndpointBasePath(restEndpoint: string) { - const globalConfig = Container.get(GlobalConfig); - - let path = globalConfig.taskRunners.path; - if (path.startsWith('/')) { - path = path.slice(1); - } - if (path.endsWith('/')) { - path = path.slice(-1); - } - - return `/${restEndpoint}/${path}`; -} - -function getWsEndpoint(restEndpoint: string) { - return `${getEndpointBasePath(restEndpoint)}/_ws`; -} - @Service() export class TaskRunnerService { runnerConnections: Map = new Map(); @@ -49,6 +23,7 @@ export class TaskRunnerService { constructor( private readonly logger: Logger, private readonly taskBroker: TaskBroker, + private readonly disconnectAnalyzer: TaskRunnerDisconnectAnalyzer, ) {} sendMessage(id: TaskRunner['id'], message: N8nMessage.ToRunner.All) { @@ -61,7 +36,7 @@ export class TaskRunnerService { let isConnected = false; - const onMessage = (data: WebSocket.RawData) => { + const onMessage = async (data: WebSocket.RawData) => { try { const buffer = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data); @@ -72,7 +47,7 @@ export class TaskRunnerService { if (!isConnected && message.type !== 'runner:info') { return; } else if (!isConnected && message.type === 'runner:info') { - this.removeConnection(id); + await this.removeConnection(id); isConnected = true; this.runnerConnections.set(id, connection); @@ -87,8 +62,6 @@ export class TaskRunnerService { this.sendMessage.bind(this, id) as MessageCallback, ); - this.sendMessage(id, { type: 'broker:runnerregistered' }); - this.logger.info(`Runner "${message.name}"(${id}) has been registered`); return; } @@ -104,10 +77,10 @@ export class TaskRunnerService { }; // Makes sure to remove the session if the connection is closed - connection.once('close', () => { + connection.once('close', async () => { connection.off('pong', heartbeat); connection.off('message', onMessage); - this.removeConnection(id); + await this.removeConnection(id); }); connection.on('message', onMessage); @@ -116,10 +89,11 @@ export class TaskRunnerService { ); } - removeConnection(id: TaskRunner['id']) { + async removeConnection(id: TaskRunner['id']) { const connection = this.runnerConnections.get(id); if (connection) { - this.taskBroker.deregisterRunner(id); + const disconnectReason = await this.disconnectAnalyzer.determineDisconnectReason(id); + this.taskBroker.deregisterRunner(id, disconnectReason); connection.close(); this.runnerConnections.delete(id); } @@ -129,61 +103,3 @@ export class TaskRunnerService { this.add(req.query.id, req.ws); } } - -// Checks for upgrade requests on the runners path and upgrades the connection -// then, passes the request back to the app to handle the routing -export const setupRunnerServer = (restEndpoint: string, server: Server, app: Application) => { - const globalConfig = Container.get(GlobalConfig); - const { authToken } = globalConfig.taskRunners; - - if (!authToken) { - throw new ApplicationError( - 'Authentication token must be configured when task runners are enabled. Use N8N_RUNNERS_AUTH_TOKEN environment variable to set it.', - ); - } - - const endpoint = getWsEndpoint(restEndpoint); - const wsServer = new WSServer({ noServer: true }); - server.on('upgrade', (request: TaskRunnerServerInitRequest, socket: Socket, head) => { - if (parseUrl(request.url).pathname !== endpoint) { - // We can't close the connection here since the Push connections - // are using the same HTTP server and upgrade requests and this - // gets triggered for both - return; - } - - wsServer.handleUpgrade(request, socket, head, (ws) => { - request.ws = ws; - - const response = new ServerResponse(request); - response.writeHead = (statusCode) => { - if (statusCode > 200) ws.close(); - return response; - }; - - // @ts-expect-error Hidden API? - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - app.handle(request, response); - }); - }); -}; - -export const setupRunnerHandler = (restEndpoint: string, app: Application) => { - const wsEndpoint = getWsEndpoint(restEndpoint); - const authEndpoint = `${getEndpointBasePath(restEndpoint)}/auth`; - - const taskRunnerAuthController = Container.get(TaskRunnerAuthController); - const taskRunnerService = Container.get(TaskRunnerService); - app.use( - wsEndpoint, - // eslint-disable-next-line @typescript-eslint/unbound-method - taskRunnerAuthController.authMiddleware, - (req: TaskRunnerServerInitRequest, res: TaskRunnerServerInitResponse) => - taskRunnerService.handleRequest(req, res), - ); - - app.post( - authEndpoint, - send(async (req) => await taskRunnerAuthController.createGrantToken(req)), - ); -}; diff --git a/packages/cli/src/runners/sliding-window-signal.ts b/packages/cli/src/runners/sliding-window-signal.ts new file mode 100644 index 0000000000..5954f7bade --- /dev/null +++ b/packages/cli/src/runners/sliding-window-signal.ts @@ -0,0 +1,59 @@ +import type { TypedEmitter } from '../typed-emitter'; + +export type SlidingWindowSignalOpts = { + windowSizeInMs?: number; +}; + +/** + * A class that listens for a specific event on an emitter (signal) and + * provides a sliding window of the last event that was emitted. + */ +export class SlidingWindowSignal { + private lastSignal: TEvents[TEventName] | null = null; + + private lastSignalTime: number = 0; + + private windowSizeInMs: number; + + constructor( + private readonly eventEmitter: TypedEmitter, + private readonly eventName: TEventName, + opts: SlidingWindowSignalOpts = {}, + ) { + const { windowSizeInMs = 500 } = opts; + + this.windowSizeInMs = windowSizeInMs; + + eventEmitter.on(eventName, (signal: TEvents[TEventName]) => { + this.lastSignal = signal; + this.lastSignalTime = Date.now(); + }); + } + + /** + * If an event has been emitted within the last `windowSize` milliseconds, + * that event is returned. Otherwise it will wait for up to `windowSize` + * milliseconds for the event to be emitted. `null` is returned + * if no event is emitted within the window. + */ + public async getSignal(): Promise { + const timeSinceLastEvent = Date.now() - this.lastSignalTime; + if (timeSinceLastEvent <= this.windowSizeInMs) return this.lastSignal; + + return await new Promise((resolve) => { + let timeoutTimerId: NodeJS.Timeout | null = null; + + const onExit = (signal: TEvents[TEventName]) => { + if (timeoutTimerId) clearTimeout(timeoutTimerId); + resolve(signal); + }; + + timeoutTimerId = setTimeout(() => { + this.eventEmitter.off(this.eventName, onExit); + resolve(null); + }); + + this.eventEmitter.once(this.eventName, onExit); + }); + } +} diff --git a/packages/cli/src/runners/task-broker.service.ts b/packages/cli/src/runners/task-broker.service.ts index 829910b468..d88d677725 100644 --- a/packages/cli/src/runners/task-broker.service.ts +++ b/packages/cli/src/runners/task-broker.service.ts @@ -2,6 +2,7 @@ import { ApplicationError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; import { Service } from 'typedi'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { Logger } from '@/logging/logger.service'; import { TaskRejectError } from './errors'; @@ -71,27 +72,58 @@ export class TaskBroker { private pendingTaskRequests: TaskRequest[] = []; - constructor(private readonly logger: Logger) {} + constructor( + private readonly logger: Logger, + private readonly loadNodesAndCredentials: LoadNodesAndCredentials, + ) { + this.loadNodesAndCredentials.addPostProcessor(this.updateNodeTypes); + } + + updateNodeTypes = async () => { + await this.messageAllRunners({ + type: 'broker:nodetypes', + nodeTypes: this.loadNodesAndCredentials.types.nodes, + }); + }; expireTasks() { const now = process.hrtime.bigint(); - const invalidOffers: number[] = []; - for (let i = 0; i < this.pendingTaskOffers.length; i++) { + for (let i = this.pendingTaskOffers.length - 1; i >= 0; i--) { if (this.pendingTaskOffers[i].validUntil < now) { - invalidOffers.push(i); + this.pendingTaskOffers.splice(i, 1); } } - - // We reverse the list so the later indexes are valid after deleting earlier ones - invalidOffers.reverse().forEach((i) => this.pendingTaskOffers.splice(i, 1)); } registerRunner(runner: TaskRunner, messageCallback: MessageCallback) { this.knownRunners.set(runner.id, { runner, messageCallback }); + void this.knownRunners.get(runner.id)!.messageCallback({ type: 'broker:runnerregistered' }); + void this.knownRunners.get(runner.id)!.messageCallback({ + type: 'broker:nodetypes', + nodeTypes: this.loadNodesAndCredentials.types.nodes, + }); } - deregisterRunner(runnerId: string) { + deregisterRunner(runnerId: string, error: Error) { this.knownRunners.delete(runnerId); + + // Remove any pending offers + for (let i = this.pendingTaskOffers.length - 1; i >= 0; i--) { + if (this.pendingTaskOffers[i].runnerId === runnerId) { + this.pendingTaskOffers.splice(i, 1); + } + } + + // Fail any tasks + for (const task of this.tasks.values()) { + if (task.runnerId === runnerId) { + void this.failTask(task.id, error); + this.handleRunnerReject( + task.id, + `The Task Runner (${runnerId}) has disconnected: ${error.message}`, + ); + } + } } registerRequester(requesterId: string, messageCallback: RequesterMessageCallback) { @@ -106,6 +138,14 @@ export class TaskBroker { await this.knownRunners.get(runnerId)?.messageCallback(message); } + private async messageAllRunners(message: N8nMessage.ToRunner.All) { + await Promise.allSettled( + [...this.knownRunners.values()].map(async (runner) => { + await runner.messageCallback(message); + }), + ); + } + private async messageRequester(requesterId: string, message: N8nMessage.ToRequester.All) { await this.requesters.get(requesterId)?.(message); } @@ -315,7 +355,7 @@ export class TaskBroker { }); } - private async failTask(taskId: Task['id'], reason: string) { + private async failTask(taskId: Task['id'], error: Error) { const task = this.tasks.get(taskId); if (!task) { return; @@ -325,7 +365,7 @@ export class TaskBroker { await this.messageRequester(task.requesterId, { type: 'broker:taskerror', taskId, - error: reason, + error, }); } @@ -338,11 +378,14 @@ export class TaskBroker { } const runner = this.knownRunners.get(task.runnerId); if (!runner) { - const reason = `Cannot find runner, failed to find runner (${task.runnerId})`; - await this.failTask(taskId, reason); - throw new ApplicationError(reason, { - level: 'error', - }); + const error = new ApplicationError( + `Cannot find runner, failed to find runner (${task.runnerId})`, + { + level: 'error', + }, + ); + await this.failTask(taskId, error); + throw error; } return runner.runner; } diff --git a/packages/cli/src/runners/task-managers/single-main-task-manager.ts b/packages/cli/src/runners/task-managers/local-task-manager.ts similarity index 92% rename from packages/cli/src/runners/task-managers/single-main-task-manager.ts rename to packages/cli/src/runners/task-managers/local-task-manager.ts index b5b60df72b..a8fca01b2c 100644 --- a/packages/cli/src/runners/task-managers/single-main-task-manager.ts +++ b/packages/cli/src/runners/task-managers/local-task-manager.ts @@ -5,7 +5,7 @@ import type { RequesterMessage } from '../runner-types'; import type { RequesterMessageCallback } from '../task-broker.service'; import { TaskBroker } from '../task-broker.service'; -export class SingleMainTaskManager extends TaskManager { +export class LocalTaskManager extends TaskManager { taskBroker: TaskBroker; id: string = 'single-main'; diff --git a/packages/cli/src/runners/task-runner-disconnect-analyzer.ts b/packages/cli/src/runners/task-runner-disconnect-analyzer.ts new file mode 100644 index 0000000000..d75a1b9aad --- /dev/null +++ b/packages/cli/src/runners/task-runner-disconnect-analyzer.ts @@ -0,0 +1,60 @@ +import { TaskRunnersConfig } from '@n8n/config'; +import { Service } from 'typedi'; + +import config from '@/config'; + +import { TaskRunnerDisconnectedError } from './errors/task-runner-disconnected-error'; +import { TaskRunnerOomError } from './errors/task-runner-oom-error'; +import { SlidingWindowSignal } from './sliding-window-signal'; +import type { TaskRunner } from './task-broker.service'; +import type { ExitReason, TaskRunnerProcessEventMap } from './task-runner-process'; +import { TaskRunnerProcess } from './task-runner-process'; + +/** + * Analyzes the disconnect reason of a task runner process to provide a more + * meaningful error message to the user. + */ +@Service() +export class TaskRunnerDisconnectAnalyzer { + private readonly exitReasonSignal: SlidingWindowSignal; + + constructor( + private readonly runnerConfig: TaskRunnersConfig, + private readonly taskRunnerProcess: TaskRunnerProcess, + ) { + // When the task runner process is running as a child process, there's + // no determinate time when it exits compared to when the runner disconnects + // (i.e. it's a race condition). Hence we use a sliding window to determine + // the exit reason. As long as we receive the exit signal from the task + // runner process within the window, we can determine the exit reason. + this.exitReasonSignal = new SlidingWindowSignal(this.taskRunnerProcess, 'exit', { + windowSizeInMs: 500, + }); + } + + private get isCloudDeployment() { + return config.get('deployment.type') === 'cloud'; + } + + async determineDisconnectReason(runnerId: TaskRunner['id']): Promise { + const exitCode = await this.awaitExitSignal(); + if (exitCode === 'oom') { + return new TaskRunnerOomError(runnerId, this.isCloudDeployment); + } + + return new TaskRunnerDisconnectedError(runnerId); + } + + private async awaitExitSignal(): Promise { + if (this.runnerConfig.mode === 'external') { + // If the task runner is running in external mode, we don't have + // control over the process and hence cannot determine the exit + // reason. We just return 'unknown' in this case. + return 'unknown'; + } + + const lastExitReason = await this.exitReasonSignal.getSignal(); + + return lastExitReason?.reason ?? 'unknown'; + } +} diff --git a/packages/cli/src/runners/task-runner-process.ts b/packages/cli/src/runners/task-runner-process.ts index 9f570fcb38..5b31a96ba3 100644 --- a/packages/cli/src/runners/task-runner-process.ts +++ b/packages/cli/src/runners/task-runner-process.ts @@ -1,19 +1,32 @@ -import { GlobalConfig } from '@n8n/config'; +import { TaskRunnersConfig } from '@n8n/config'; import * as a from 'node:assert/strict'; import { spawn } from 'node:child_process'; import * as process from 'node:process'; import { Service } from 'typedi'; +import { OnShutdown } from '@/decorators/on-shutdown'; +import { Logger } from '@/logging/logger.service'; + import { TaskRunnerAuthService } from './auth/task-runner-auth.service'; -import { OnShutdown } from '../decorators/on-shutdown'; +import { forwardToLogger } from './forward-to-logger'; +import { NodeProcessOomDetector } from './node-process-oom-detector'; +import { TypedEmitter } from '../typed-emitter'; type ChildProcess = ReturnType; +export type ExitReason = 'unknown' | 'oom'; + +export type TaskRunnerProcessEventMap = { + exit: { + reason: ExitReason; + }; +}; + /** * Manages the JS task runner process as a child process */ @Service() -export class TaskRunnerProcess { +export class TaskRunnerProcess extends TypedEmitter { public get isRunning() { return this.process !== null; } @@ -23,40 +36,79 @@ export class TaskRunnerProcess { return this.process?.pid; } + /** Promise that resolves when the process has exited */ + public get runPromise() { + return this._runPromise; + } + + private get useLauncher() { + return this.runnerConfig.mode === 'internal_launcher'; + } + private process: ChildProcess | null = null; - /** Promise that resolves after the process has exited */ - private runPromise: Promise | null = null; + private _runPromise: Promise | null = null; + + private oomDetector: NodeProcessOomDetector | null = null; private isShuttingDown = false; + private logger: Logger; + + private readonly passthroughEnvVars = [ + 'PATH', + 'NODE_FUNCTION_ALLOW_BUILTIN', + 'NODE_FUNCTION_ALLOW_EXTERNAL', + ] as const; + constructor( - private readonly globalConfig: GlobalConfig, + logger: Logger, + private readonly runnerConfig: TaskRunnersConfig, private readonly authService: TaskRunnerAuthService, - ) {} + ) { + super(); + + a.ok( + this.runnerConfig.mode === 'internal_childprocess' || + this.runnerConfig.mode === 'internal_launcher', + ); + + this.logger = logger.scoped('task-runner'); + } async start() { a.ok(!this.process, 'Task Runner Process already running'); const grantToken = await this.authService.createGrantToken(); - const startScript = require.resolve('@n8n/task-runner'); - this.process = spawn('node', [startScript], { - env: { - PATH: process.env.PATH, - N8N_RUNNERS_GRANT_TOKEN: grantToken, - N8N_RUNNERS_N8N_URI: `127.0.0.1:${this.globalConfig.taskRunners.port}`, - NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, - NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, - }, - }); + const n8nUri = `127.0.0.1:${this.runnerConfig.port}`; + this.process = this.useLauncher + ? this.startLauncher(grantToken, n8nUri) + : this.startNode(grantToken, n8nUri); - this.process.stdout?.pipe(process.stdout); - this.process.stderr?.pipe(process.stderr); + forwardToLogger(this.logger, this.process, '[Task Runner]: '); this.monitorProcess(this.process); } + startNode(grantToken: string, n8nUri: string) { + const startScript = require.resolve('@n8n/task-runner'); + + return spawn('node', [startScript], { + env: this.getProcessEnvVars(grantToken, n8nUri), + }); + } + + startLauncher(grantToken: string, n8nUri: string) { + return spawn(this.runnerConfig.launcherPath, ['launch', this.runnerConfig.launcherRunner], { + env: { + ...this.getProcessEnvVars(grantToken, n8nUri), + // For debug logging if enabled + RUST_LOG: process.env.RUST_LOG, + }, + }); + } + @OnShutdown() async stop() { if (!this.process) { @@ -66,14 +118,45 @@ export class TaskRunnerProcess { this.isShuttingDown = true; // TODO: Timeout & force kill - this.process.kill(); - await this.runPromise; + if (this.useLauncher) { + await this.killLauncher(); + } else { + this.killNode(); + } + await this._runPromise; this.isShuttingDown = false; } + killNode() { + if (!this.process) { + return; + } + this.process.kill(); + } + + async killLauncher() { + if (!this.process?.pid) { + return; + } + + const killProcess = spawn(this.runnerConfig.launcherPath, [ + 'kill', + this.runnerConfig.launcherRunner, + this.process.pid.toString(), + ]); + + await new Promise((resolve) => { + killProcess.on('exit', () => { + resolve(); + }); + }); + } + private monitorProcess(taskRunnerProcess: ChildProcess) { - this.runPromise = new Promise((resolve) => { + this._runPromise = new Promise((resolve) => { + this.oomDetector = new NodeProcessOomDetector(taskRunnerProcess); + taskRunnerProcess.on('exit', (code) => { this.onProcessExit(code, resolve); }); @@ -82,6 +165,7 @@ export class TaskRunnerProcess { private onProcessExit(_code: number | null, resolveFn: () => void) { this.process = null; + this.emit('exit', { reason: this.oomDetector?.didProcessOom ? 'oom' : 'unknown' }); resolveFn(); // If we are not shutting down, restart the process @@ -89,4 +173,30 @@ export class TaskRunnerProcess { setImmediate(async () => await this.start()); } } + + private getProcessEnvVars(grantToken: string, n8nUri: string) { + const envVars: Record = { + N8N_RUNNERS_GRANT_TOKEN: grantToken, + N8N_RUNNERS_N8N_URI: n8nUri, + N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(), + N8N_RUNNERS_MAX_CONCURRENCY: this.runnerConfig.maxConcurrency.toString(), + ...this.getPassthroughEnvVars(), + }; + + if (this.runnerConfig.maxOldSpaceSize) { + envVars.NODE_OPTIONS = `--max-old-space-size=${this.runnerConfig.maxOldSpaceSize}`; + } + + return envVars; + } + + private getPassthroughEnvVars() { + return this.passthroughEnvVars.reduce>((env, key) => { + if (process.env[key]) { + env[key] = process.env[key]; + } + + return env; + }, {}); + } } diff --git a/packages/cli/src/runners/task-runner-server.ts b/packages/cli/src/runners/task-runner-server.ts index fc31c100a3..2199e70b38 100644 --- a/packages/cli/src/runners/task-runner-server.ts +++ b/packages/cli/src/runners/task-runner-server.ts @@ -88,7 +88,7 @@ export class TaskRunnerServer { this.server = createHttpServer(app); const { - taskRunners: { port, listen_address: address }, + taskRunners: { port, listenAddress: address }, } = this.globalConfig; this.server.on('error', (error: Error & { code: string }) => { @@ -114,7 +114,10 @@ export class TaskRunnerServer { a.ok(authToken); a.ok(this.server); - this.wsServer = new WSServer({ noServer: true }); + this.wsServer = new WSServer({ + noServer: true, + maxPayload: this.globalConfig.taskRunners.maxPayload, + }); this.server.on('upgrade', this.handleUpgradeRequest); } @@ -122,11 +125,13 @@ export class TaskRunnerServer { const { app } = this; // Augment errors sent to Sentry - const { - Handlers: { requestHandler, errorHandler }, - } = await import('@sentry/node'); - app.use(requestHandler()); - app.use(errorHandler()); + if (this.globalConfig.sentry.backendDsn) { + const { + Handlers: { requestHandler, errorHandler }, + } = await import('@sentry/node'); + app.use(requestHandler()); + app.use(errorHandler()); + } } private setupCommonMiddlewares() { diff --git a/packages/cli/src/scaling/__tests__/job-processor.service.test.ts b/packages/cli/src/scaling/__tests__/job-processor.service.test.ts new file mode 100644 index 0000000000..6a3fa5caa4 --- /dev/null +++ b/packages/cli/src/scaling/__tests__/job-processor.service.test.ts @@ -0,0 +1,21 @@ +import { mock } from 'jest-mock-extended'; + +import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { IExecutionResponse } from '@/interfaces'; + +import { JobProcessor } from '../job-processor'; +import type { Job } from '../scaling.types'; + +describe('JobProcessor', () => { + it('should refrain from processing a crashed execution', async () => { + const executionRepository = mock(); + executionRepository.findSingleExecution.mockResolvedValue( + mock({ status: 'crashed' }), + ); + const jobProcessor = new JobProcessor(mock(), executionRepository, mock(), mock(), mock()); + + const result = await jobProcessor.processJob(mock()); + + expect(result).toEqual({ success: false }); + }); +}); diff --git a/packages/cli/src/scaling/__tests__/publisher.service.test.ts b/packages/cli/src/scaling/__tests__/publisher.service.test.ts index 05bb52bc6a..f69ad08cb5 100644 --- a/packages/cli/src/scaling/__tests__/publisher.service.test.ts +++ b/packages/cli/src/scaling/__tests__/publisher.service.test.ts @@ -1,35 +1,35 @@ import type { Redis as SingleNodeClient } from 'ioredis'; import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; import config from '@/config'; -import { generateNanoId } from '@/databases/utils/generators'; import type { RedisClientService } from '@/services/redis-client.service'; +import { mockLogger } from '@test/mocking'; import { Publisher } from '../pubsub/publisher.service'; import type { PubSub } from '../pubsub/pubsub.types'; describe('Publisher', () => { - let queueModeId: string; - beforeEach(() => { config.set('executions.mode', 'queue'); - queueModeId = generateNanoId(); - config.set('redis.queueModeId', queueModeId); }); const client = mock(); + const logger = mockLogger(); + const hostId = 'main-bnxa1riryKUNHtln'; + const instanceSettings = mock({ hostId }); const redisClientService = mock({ createClient: () => client }); describe('constructor', () => { it('should init Redis client in scaling mode', () => { - const publisher = new Publisher(mock(), redisClientService); + const publisher = new Publisher(logger, redisClientService, instanceSettings); expect(publisher.getClient()).toEqual(client); }); it('should not init Redis client in regular mode', () => { config.set('executions.mode', 'regular'); - const publisher = new Publisher(mock(), redisClientService); + const publisher = new Publisher(logger, redisClientService, instanceSettings); expect(publisher.getClient()).toBeUndefined(); }); @@ -37,29 +37,39 @@ describe('Publisher', () => { describe('shutdown', () => { it('should disconnect Redis client', () => { - const publisher = new Publisher(mock(), redisClientService); + const publisher = new Publisher(logger, redisClientService, instanceSettings); publisher.shutdown(); expect(client.disconnect).toHaveBeenCalled(); }); }); describe('publishCommand', () => { + it('should do nothing if not in scaling mode', async () => { + config.set('executions.mode', 'regular'); + const publisher = new Publisher(logger, redisClientService, instanceSettings); + const msg = mock({ command: 'reload-license' }); + + await publisher.publishCommand(msg); + + expect(client.publish).not.toHaveBeenCalled(); + }); + it('should publish command into `n8n.commands` pubsub channel', async () => { - const publisher = new Publisher(mock(), redisClientService); + const publisher = new Publisher(logger, redisClientService, instanceSettings); const msg = mock({ command: 'reload-license' }); await publisher.publishCommand(msg); expect(client.publish).toHaveBeenCalledWith( 'n8n.commands', - JSON.stringify({ ...msg, senderId: queueModeId, selfSend: false, debounce: true }), + JSON.stringify({ ...msg, senderId: hostId, selfSend: false, debounce: true }), ); }); }); describe('publishWorkerResponse', () => { it('should publish worker response into `n8n.worker-response` pubsub channel', async () => { - const publisher = new Publisher(mock(), redisClientService); + const publisher = new Publisher(logger, redisClientService, instanceSettings); const msg = mock({ response: 'response-to-get-worker-status', }); diff --git a/packages/cli/src/scaling/__tests__/scaling.service.test.ts b/packages/cli/src/scaling/__tests__/scaling.service.test.ts index a6c14ab964..0b5f80da48 100644 --- a/packages/cli/src/scaling/__tests__/scaling.service.test.ts +++ b/packages/cli/src/scaling/__tests__/scaling.service.test.ts @@ -56,6 +56,7 @@ describe('ScalingService', () => { let registerWorkerListenersSpy: jest.SpyInstance; let scheduleQueueRecoverySpy: jest.SpyInstance; let stopQueueRecoverySpy: jest.SpyInstance; + let stopQueueMetricsSpy: jest.SpyInstance; let getRunningJobsCountSpy: jest.SpyInstance; const bullConstructorArgs = [ @@ -99,6 +100,9 @@ describe('ScalingService', () => { scheduleQueueRecoverySpy = jest.spyOn(scalingService, 'scheduleQueueRecovery'); // @ts-expect-error Private method stopQueueRecoverySpy = jest.spyOn(scalingService, 'stopQueueRecovery'); + + // @ts-expect-error Private method + stopQueueMetricsSpy = jest.spyOn(scalingService, 'stopQueueMetrics'); }); describe('setupQueue', () => { @@ -180,15 +184,37 @@ describe('ScalingService', () => { }); describe('stop', () => { - it('should pause queue, wait for running jobs, stop queue recovery', async () => { - await scalingService.setupQueue(); - jobProcessor.getRunningJobIds.mockReturnValue([]); + describe('if main', () => { + it('should pause queue, stop queue recovery and queue metrics', async () => { + // @ts-expect-error readonly property + instanceSettings.instanceType = 'main'; + await scalingService.setupQueue(); + // @ts-expect-error readonly property + scalingService.queueRecoveryContext.timeout = 1; + jest.spyOn(scalingService, 'isQueueMetricsEnabled', 'get').mockReturnValue(true); - await scalingService.stop(); + await scalingService.stop(); - expect(queue.pause).toHaveBeenCalledWith(true, true); - expect(stopQueueRecoverySpy).toHaveBeenCalled(); - expect(getRunningJobsCountSpy).toHaveBeenCalled(); + expect(getRunningJobsCountSpy).not.toHaveBeenCalled(); + expect(queue.pause).toHaveBeenCalledWith(true, true); + expect(stopQueueRecoverySpy).toHaveBeenCalled(); + expect(stopQueueMetricsSpy).toHaveBeenCalled(); + }); + }); + + describe('if worker', () => { + it('should wait for running jobs to finish', async () => { + // @ts-expect-error readonly property + instanceSettings.instanceType = 'worker'; + await scalingService.setupQueue(); + jobProcessor.getRunningJobIds.mockReturnValue([]); + + await scalingService.stop(); + + expect(getRunningJobsCountSpy).toHaveBeenCalled(); + expect(queue.pause).not.toHaveBeenCalled(); + expect(stopQueueRecoverySpy).not.toHaveBeenCalled(); + }); }); }); diff --git a/packages/cli/src/scaling/__tests__/subscriber.service.test.ts b/packages/cli/src/scaling/__tests__/subscriber.service.test.ts index 62834dba33..4f97208b99 100644 --- a/packages/cli/src/scaling/__tests__/subscriber.service.test.ts +++ b/packages/cli/src/scaling/__tests__/subscriber.service.test.ts @@ -17,14 +17,14 @@ describe('Subscriber', () => { describe('constructor', () => { it('should init Redis client in scaling mode', () => { - const subscriber = new Subscriber(mock(), redisClientService, mock()); + const subscriber = new Subscriber(mock(), redisClientService, mock(), mock()); expect(subscriber.getClient()).toEqual(client); }); it('should not init Redis client in regular mode', () => { config.set('executions.mode', 'regular'); - const subscriber = new Subscriber(mock(), redisClientService, mock()); + const subscriber = new Subscriber(mock(), redisClientService, mock(), mock()); expect(subscriber.getClient()).toBeUndefined(); }); @@ -32,7 +32,7 @@ describe('Subscriber', () => { describe('shutdown', () => { it('should disconnect Redis client', () => { - const subscriber = new Subscriber(mock(), redisClientService, mock()); + const subscriber = new Subscriber(mock(), redisClientService, mock(), mock()); subscriber.shutdown(); expect(client.disconnect).toHaveBeenCalled(); }); @@ -40,7 +40,7 @@ describe('Subscriber', () => { describe('subscribe', () => { it('should subscribe to pubsub channel', async () => { - const subscriber = new Subscriber(mock(), redisClientService, mock()); + const subscriber = new Subscriber(mock(), redisClientService, mock(), mock()); await subscriber.subscribe('n8n.commands'); diff --git a/packages/cli/src/scaling/__tests__/worker-server.test.ts b/packages/cli/src/scaling/__tests__/worker-server.test.ts index 778d403bf2..8bcdd3aa5c 100644 --- a/packages/cli/src/scaling/__tests__/worker-server.test.ts +++ b/packages/cli/src/scaling/__tests__/worker-server.test.ts @@ -8,6 +8,7 @@ import * as http from 'node:http'; import type { ExternalHooks } from '@/external-hooks'; import type { PrometheusMetricsService } from '@/metrics/prometheus-metrics.service'; import { bodyParser, rawBodyReader } from '@/middlewares'; +import { mockLogger } from '@test/mocking'; import { WorkerServer } from '../worker-server'; @@ -48,7 +49,7 @@ describe('WorkerServer', () => { () => new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, mock({ instanceType: 'webhook' }), @@ -73,7 +74,7 @@ describe('WorkerServer', () => { new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, instanceSettings, @@ -100,7 +101,7 @@ describe('WorkerServer', () => { const workerServer = new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, instanceSettings, @@ -135,7 +136,7 @@ describe('WorkerServer', () => { const workerServer = new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, instanceSettings, @@ -156,7 +157,7 @@ describe('WorkerServer', () => { const workerServer = new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, instanceSettings, @@ -174,7 +175,7 @@ describe('WorkerServer', () => { const workerServer = new WorkerServer( globalConfig, - mock(), + mockLogger(), mock(), externalHooks, instanceSettings, diff --git a/packages/cli/src/scaling/constants.ts b/packages/cli/src/scaling/constants.ts index 348f156896..e56596e4a0 100644 --- a/packages/cli/src/scaling/constants.ts +++ b/packages/cli/src/scaling/constants.ts @@ -20,4 +20,7 @@ export const SELF_SEND_COMMANDS = new Set([ * Commands that should not be debounced when received, e.g. during webhook handling in * multi-main setup. */ -export const IMMEDIATE_COMMANDS = new Set(['relay-execution-lifecycle-event']); +export const IMMEDIATE_COMMANDS = new Set([ + 'add-webhooks-triggers-and-pollers', + 'relay-execution-lifecycle-event', +]); diff --git a/packages/cli/src/scaling/job-processor.ts b/packages/cli/src/scaling/job-processor.ts index 49e1383ac6..6bf2524304 100644 --- a/packages/cli/src/scaling/job-processor.ts +++ b/packages/cli/src/scaling/job-processor.ts @@ -1,7 +1,12 @@ import type { RunningJobSummary } from '@n8n/api-types'; -import { WorkflowExecute } from 'n8n-core'; -import { BINARY_ENCODING, ApplicationError, Workflow } from 'n8n-workflow'; +import { InstanceSettings, WorkflowExecute } from 'n8n-core'; import type { ExecutionStatus, IExecuteResponsePromiseData, IRun } from 'n8n-workflow'; +import { + BINARY_ENCODING, + ApplicationError, + Workflow, + ErrorReporterProxy as ErrorReporter, +} from 'n8n-workflow'; import type PCancelable from 'p-cancelable'; import { Service } from 'typedi'; @@ -12,7 +17,14 @@ import { Logger } from '@/logging/logger.service'; import { NodeTypes } from '@/node-types'; import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; -import type { Job, JobId, JobResult, RunningJob } from './scaling.types'; +import type { + Job, + JobFinishedMessage, + JobId, + JobResult, + RespondToWebhookMessage, + RunningJob, +} from './scaling.types'; /** * Responsible for processing jobs from the queue, i.e. running enqueued executions. @@ -26,7 +38,10 @@ export class JobProcessor { private readonly executionRepository: ExecutionRepository, private readonly workflowRepository: WorkflowRepository, private readonly nodeTypes: NodeTypes, - ) {} + private readonly instanceSettings: InstanceSettings, + ) { + this.logger = this.logger.scoped('scaling'); + } async processJob(job: Job): Promise { const { executionId, loadStaticData } = job.data; @@ -37,15 +52,25 @@ export class JobProcessor { }); if (!execution) { - this.logger.error('[JobProcessor] Failed to find execution data', { executionId }); - throw new ApplicationError('Failed to find execution data. Aborting execution.', { - extra: { executionId }, - }); + throw new ApplicationError( + `Worker failed to find data for execution ${executionId} (job ${job.id})`, + { level: 'warning' }, + ); } + /** + * Bull's implicit retry mechanism and n8n's execution recovery mechanism may + * cause a crashed execution to be enqueued. We refrain from processing it, + * until we have reworked both mechanisms to prevent this scenario. + */ + if (execution.status === 'crashed') return { success: false }; + const workflowId = execution.workflowData.id; - this.logger.info(`[JobProcessor] Starting job ${job.id} (execution ${executionId})`); + this.logger.info(`Worker started execution ${executionId} (job ${job.id})`, { + executionId, + jobId: job.id, + }); const startedAt = await this.executionRepository.setRunning(executionId); @@ -58,8 +83,10 @@ export class JobProcessor { }); if (workflowData === null) { - this.logger.error('[JobProcessor] Failed to find workflow', { workflowId, executionId }); - throw new ApplicationError('Failed to find workflow', { extra: { workflowId } }); + throw new ApplicationError( + `Worker failed to find workflow ${workflowId} to run execution ${executionId} (job ${job.id})`, + { level: 'warning' }, + ); } staticData = workflowData.staticData; @@ -102,11 +129,14 @@ export class JobProcessor { additionalData.hooks.hookFunctions.sendResponse = [ async (response: IExecuteResponsePromiseData): Promise => { - await job.progress({ + const msg: RespondToWebhookMessage = { kind: 'respond-to-webhook', executionId, response: this.encodeWebhookResponse(response), - }); + workerId: this.instanceSettings.hostId, + }; + + await job.progress(msg); }, ]; @@ -115,7 +145,7 @@ export class JobProcessor { additionalData.setExecutionStatus = (status: ExecutionStatus) => { // Can't set the status directly in the queued worker, but it will happen in InternalHook.onWorkflowPostExecute this.logger.debug( - `[JobProcessor] Queued worker execution status for ${executionId} is "${status}"`, + `Queued worker execution status for execution ${executionId} (job ${job.id}) is "${status}"`, ); }; @@ -125,6 +155,7 @@ export class JobProcessor { workflowExecute = new WorkflowExecute(additionalData, execution.mode, execution.data); workflowRun = workflowExecute.processRunExecutionData(workflow); } else { + ErrorReporter.info(`Worker found execution ${executionId} without data`); // Execute all nodes // Can execute without webhook so go on workflowExecute = new WorkflowExecute(additionalData, execution.mode); @@ -148,7 +179,18 @@ export class JobProcessor { delete this.runningJobs[job.id]; - this.logger.debug('[JobProcessor] Job finished running', { jobId: job.id, executionId }); + this.logger.info(`Worker finished execution ${executionId} (job ${job.id})`, { + executionId, + jobId: job.id, + }); + + const msg: JobFinishedMessage = { + kind: 'job-finished', + executionId, + workerId: this.instanceSettings.hostId, + }; + + await job.progress(msg); /** * @important Do NOT call `workflowExecuteAfter` hook here. diff --git a/packages/cli/src/services/orchestration/main/multi-main-setup.ee.ts b/packages/cli/src/scaling/multi-main-setup.ee.ts similarity index 62% rename from packages/cli/src/services/orchestration/main/multi-main-setup.ee.ts rename to packages/cli/src/scaling/multi-main-setup.ee.ts index bb1b52519c..8be7f4ae51 100644 --- a/packages/cli/src/services/orchestration/main/multi-main-setup.ee.ts +++ b/packages/cli/src/scaling/multi-main-setup.ee.ts @@ -1,5 +1,5 @@ +import { GlobalConfig } from '@n8n/config'; import { InstanceSettings } from 'n8n-core'; -import { ErrorReporterProxy as EventReporter } from 'n8n-workflow'; import { Service } from 'typedi'; import config from '@/config'; @@ -10,10 +10,22 @@ import { RedisClientService } from '@/services/redis-client.service'; import { TypedEmitter } from '@/typed-emitter'; type MultiMainEvents = { + /** + * Emitted when this instance loses leadership. In response, its various + * services will stop triggers, pollers, pruning, wait-tracking, license + * renewal, queue recovery, etc. + */ 'leader-stepdown': never; + + /** + * Emitted when this instance gains leadership. In response, its various + * services will start triggers, pollers, pruning, wait-tracking, license + * renewal, queue recovery, etc. + */ 'leader-takeover': never; }; +/** Designates leader and followers when running multiple main processes. */ @Service() export class MultiMainSetup extends TypedEmitter { constructor( @@ -21,17 +33,15 @@ export class MultiMainSetup extends TypedEmitter { private readonly instanceSettings: InstanceSettings, private readonly publisher: Publisher, private readonly redisClientService: RedisClientService, + private readonly globalConfig: GlobalConfig, ) { super(); - } - - get instanceId() { - return config.getEnv('redis.queueModeId'); + this.logger = this.logger.scoped(['scaling', 'multi-main-setup']); } private leaderKey: string; - private readonly leaderKeyTtl = config.getEnv('multiMainSetup.ttl'); + private readonly leaderKeyTtl = this.globalConfig.multiMainSetup.ttl; private leaderCheckInterval: NodeJS.Timer | undefined; @@ -44,7 +54,7 @@ export class MultiMainSetup extends TypedEmitter { this.leaderCheckInterval = setInterval(async () => { await this.checkLeader(); - }, config.getEnv('multiMainSetup.interval') * TIME.SECOND); + }, this.globalConfig.multiMainSetup.interval * TIME.SECOND); } async shutdown() { @@ -58,23 +68,25 @@ export class MultiMainSetup extends TypedEmitter { private async checkLeader() { const leaderId = await this.publisher.get(this.leaderKey); - if (leaderId === this.instanceId) { - this.logger.debug(`[Instance ID ${this.instanceId}] Leader is this instance`); + const { hostId } = this.instanceSettings; + + if (leaderId === hostId) { + this.logger.debug(`[Instance ID ${hostId}] Leader is this instance`); await this.publisher.setExpiration(this.leaderKey, this.leaderKeyTtl); return; } - if (leaderId && leaderId !== this.instanceId) { - this.logger.debug(`[Instance ID ${this.instanceId}] Leader is other instance "${leaderId}"`); + if (leaderId && leaderId !== hostId) { + this.logger.debug(`[Instance ID ${hostId}] Leader is other instance "${leaderId}"`); if (this.instanceSettings.isLeader) { this.instanceSettings.markAsFollower(); - this.emit('leader-stepdown'); // lost leadership - stop triggers, pollers, pruning, wait-tracking, queue recovery + this.emit('leader-stepdown'); - EventReporter.info('[Multi-main setup] Leader failed to renew leader key'); + this.logger.warn('[Multi-main setup] Leader failed to renew leader key'); } return; @@ -82,14 +94,11 @@ export class MultiMainSetup extends TypedEmitter { if (!leaderId) { this.logger.debug( - `[Instance ID ${this.instanceId}] Leadership vacant, attempting to become leader...`, + `[Instance ID ${hostId}] Leadership vacant, attempting to become leader...`, ); this.instanceSettings.markAsFollower(); - /** - * Lost leadership - stop triggers, pollers, pruning, wait tracking, license renewal, queue recovery - */ this.emit('leader-stepdown'); await this.tryBecomeLeader(); @@ -97,19 +106,18 @@ export class MultiMainSetup extends TypedEmitter { } private async tryBecomeLeader() { + const { hostId } = this.instanceSettings; + // this can only succeed if leadership is currently vacant - const keySetSuccessfully = await this.publisher.setIfNotExists(this.leaderKey, this.instanceId); + const keySetSuccessfully = await this.publisher.setIfNotExists(this.leaderKey, hostId); if (keySetSuccessfully) { - this.logger.debug(`[Instance ID ${this.instanceId}] Leader is now this instance`); + this.logger.debug(`[Instance ID ${hostId}] Leader is now this instance`); this.instanceSettings.markAsLeader(); await this.publisher.setExpiration(this.leaderKey, this.leaderKeyTtl); - /** - * Gained leadership - start triggers, pollers, pruning, wait-tracking, license renewal, queue recovery - */ this.emit('leader-takeover'); } else { this.instanceSettings.markAsFollower(); diff --git a/packages/cli/src/scaling/pubsub/publisher.service.ts b/packages/cli/src/scaling/pubsub/publisher.service.ts index 29d31989ff..248a455e3e 100644 --- a/packages/cli/src/scaling/pubsub/publisher.service.ts +++ b/packages/cli/src/scaling/pubsub/publisher.service.ts @@ -1,4 +1,5 @@ import type { Redis as SingleNodeClient, Cluster as MultiNodeClient } from 'ioredis'; +import { InstanceSettings } from 'n8n-core'; import { Service } from 'typedi'; import config from '@/config'; @@ -20,10 +21,13 @@ export class Publisher { constructor( private readonly logger: Logger, private readonly redisClientService: RedisClientService, + private readonly instanceSettings: InstanceSettings, ) { - // @TODO: Once this class is only ever initialized in scaling mode, throw in the next line instead. + // @TODO: Once this class is only ever initialized in scaling mode, assert in the next line. if (config.getEnv('executions.mode') !== 'queue') return; + this.logger = this.logger.scoped(['scaling', 'pubsub']); + this.client = this.redisClientService.createClient({ type: 'publisher(n8n)' }); } @@ -42,11 +46,14 @@ export class Publisher { /** Publish a command into the `n8n.commands` channel. */ async publishCommand(msg: Omit) { + // @TODO: Once this class is only ever used in scaling mode, remove next line. + if (config.getEnv('executions.mode') !== 'queue') return; + await this.client.publish( 'n8n.commands', JSON.stringify({ ...msg, - senderId: config.getEnv('redis.queueModeId'), + senderId: this.instanceSettings.hostId, selfSend: SELF_SEND_COMMANDS.has(msg.command), debounce: !IMMEDIATE_COMMANDS.has(msg.command), }), @@ -55,11 +62,11 @@ export class Publisher { this.logger.debug(`Published ${msg.command} to command channel`); } - /** Publish a response for a command into the `n8n.worker-response` channel. */ + /** Publish a response to a command into the `n8n.worker-response` channel. */ async publishWorkerResponse(msg: PubSub.WorkerResponse) { await this.client.publish('n8n.worker-response', JSON.stringify(msg)); - this.logger.debug(`Published response ${msg.response} to worker response channel`); + this.logger.debug(`Published ${msg.response} to worker response channel`); } // #endregion diff --git a/packages/cli/src/scaling/pubsub/pubsub-handler.ts b/packages/cli/src/scaling/pubsub/pubsub-handler.ts index ca590dd2c2..deeed5b584 100644 --- a/packages/cli/src/scaling/pubsub/pubsub-handler.ts +++ b/packages/cli/src/scaling/pubsub/pubsub-handler.ts @@ -3,7 +3,6 @@ import { ensureError } from 'n8n-workflow'; import { Service } from 'typedi'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; -import config from '@/config'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { EventService } from '@/events/event.service'; @@ -49,7 +48,7 @@ export class PubSubHandler { ...this.commonHandlers, 'get-worker-status': async () => await this.publisher.publishWorkerResponse({ - senderId: config.getEnv('redis.queueModeId'), + senderId: this.instanceSettings.hostId, response: 'response-to-get-worker-status', payload: this.workerStatusService.generateStatus(), }), diff --git a/packages/cli/src/scaling/pubsub/pubsub.types.ts b/packages/cli/src/scaling/pubsub/pubsub.types.ts index b4d6e1a962..eec0110201 100644 --- a/packages/cli/src/scaling/pubsub/pubsub.types.ts +++ b/packages/cli/src/scaling/pubsub/pubsub.types.ts @@ -88,7 +88,7 @@ export namespace PubSub { /** Content of worker response. */ response: WorkerResponseKey; - /** Whether the command should be debounced when received. */ + /** Whether the worker response should be debounced when received. */ debounce?: boolean; } & (PubSubWorkerResponseMap[WorkerResponseKey] extends never ? { payload?: never } // some responses carry no payload @@ -101,6 +101,10 @@ export namespace PubSub { /** Response sent via the `n8n.worker-response` pubsub channel. */ export type WorkerResponse = ToWorkerResponse<'response-to-get-worker-status'>; + // ---------------------------------- + // events + // ---------------------------------- + /** * Of all events emitted from pubsub messages, those whose handlers * are all present in main, worker, and webhook processes. diff --git a/packages/cli/src/scaling/pubsub/subscriber.service.ts b/packages/cli/src/scaling/pubsub/subscriber.service.ts index 7c7f90fb0e..ed673fc4e4 100644 --- a/packages/cli/src/scaling/pubsub/subscriber.service.ts +++ b/packages/cli/src/scaling/pubsub/subscriber.service.ts @@ -1,5 +1,6 @@ import type { Redis as SingleNodeClient, Cluster as MultiNodeClient } from 'ioredis'; import debounce from 'lodash/debounce'; +import { InstanceSettings } from 'n8n-core'; import { jsonParse } from 'n8n-workflow'; import { Service } from 'typedi'; @@ -17,16 +18,17 @@ import type { PubSub } from './pubsub.types'; export class Subscriber { private readonly client: SingleNodeClient | MultiNodeClient; - // #region Lifecycle - constructor( private readonly logger: Logger, private readonly redisClientService: RedisClientService, private readonly eventService: EventService, + private readonly instanceSettings: InstanceSettings, ) { // @TODO: Once this class is only ever initialized in scaling mode, throw in the next line instead. if (config.getEnv('executions.mode') !== 'queue') return; + this.logger = this.logger.scoped(['scaling', 'pubsub']); + this.client = this.redisClientService.createClient({ type: 'subscriber(n8n)' }); const handlerFn = (msg: PubSub.Command | PubSub.WorkerResponse) => { @@ -36,8 +38,8 @@ export class Subscriber { const debouncedHandlerFn = debounce(handlerFn, 300); - this.client.on('message', (_channel: PubSub.Channel, str) => { - const msg = this.parseMessage(str); + this.client.on('message', (channel: PubSub.Channel, str) => { + const msg = this.parseMessage(str, channel); if (!msg) return; if (msg.debounce) debouncedHandlerFn(msg); else handlerFn(msg); @@ -53,48 +55,47 @@ export class Subscriber { this.client.disconnect(); } - // #endregion - - // #region Subscribing - async subscribe(channel: PubSub.Channel) { await this.client.subscribe(channel, (error) => { if (error) { - this.logger.error('Failed to subscribe to channel', { channel, cause: error }); + this.logger.error(`Failed to subscribe to channel ${channel}`, { error }); return; } - this.logger.debug('Subscribed to channel', { channel }); + this.logger.debug(`Subscribed to channel ${channel}`); }); } - // #region Commands - - private parseMessage(str: string) { + private parseMessage(str: string, channel: PubSub.Channel) { const msg = jsonParse(str, { fallbackValue: null, }); if (!msg) { - this.logger.debug('Received invalid string via pubsub channel', { message: str }); - + this.logger.error(`Received malformed message via channel ${channel}`, { + msg: str, + channel, + }); return null; } - const queueModeId = config.getEnv('redis.queueModeId'); + const { hostId } = this.instanceSettings; if ( 'command' in msg && !msg.selfSend && - (msg.senderId === queueModeId || (msg.targets && !msg.targets.includes(queueModeId))) + (msg.senderId === hostId || (msg.targets && !msg.targets.includes(hostId))) ) { return null; } - this.logger.debug('Received message via pubsub channel', msg); + const msgName = 'command' in msg ? msg.command : msg.response; + + this.logger.debug(`Received message ${msgName} via channel ${channel}`, { + msg, + channel, + }); return msg; } - - // #endregion } diff --git a/packages/cli/src/scaling/scaling.service.ts b/packages/cli/src/scaling/scaling.service.ts index f35b4348a6..f7731e26c2 100644 --- a/packages/cli/src/scaling/scaling.service.ts +++ b/packages/cli/src/scaling/scaling.service.ts @@ -6,6 +6,7 @@ import { sleep, jsonStringify, ErrorReporterProxy, + ensureError, } from 'n8n-workflow'; import type { IExecuteResponsePromiseData } from 'n8n-workflow'; import { strict } from 'node:assert'; @@ -20,6 +21,7 @@ import { MaxStalledCountError } from '@/errors/max-stalled-count.error'; import { EventService } from '@/events/event.service'; import { Logger } from '@/logging/logger.service'; import { OrchestrationService } from '@/services/orchestration.service'; +import { assertNever } from '@/utils'; import { JOB_TYPE_NAME, QUEUE_NAME } from './constants'; import { JobProcessor } from './job-processor'; @@ -31,7 +33,8 @@ import type { JobStatus, JobId, QueueRecoveryContext, - JobReport, + JobMessage, + JobFailedMessage, } from './scaling.types'; @Service() @@ -48,7 +51,7 @@ export class ScalingService { private readonly orchestrationService: OrchestrationService, private readonly eventService: EventService, ) { - this.logger = this.logger.withScope('scaling'); + this.logger = this.logger.scoped('scaling'); } // #region Lifecycle @@ -89,34 +92,57 @@ export class ScalingService { void this.queue.process(JOB_TYPE_NAME, concurrency, async (job: Job) => { try { await this.jobProcessor.processJob(job); - } catch (error: unknown) { - // Errors thrown here will be sent to the main instance by bull. Logging - // them out and rethrowing them allows to find out which worker had the - // issue. - this.logger.error('Executing a job errored', { - jobId: job.id, - executionId: job.data.executionId, - error, - }); - ErrorReporterProxy.error(error); - throw error; + } catch (error) { + await this.reportJobProcessingError(ensureError(error), job); } }); this.logger.debug('Worker setup completed'); } + private async reportJobProcessingError(error: Error, job: Job) { + const { executionId } = job.data; + + this.logger.error(`Worker errored while running execution ${executionId} (job ${job.id})`, { + error, + executionId, + jobId: job.id, + }); + + const msg: JobFailedMessage = { + kind: 'job-failed', + executionId, + workerId: this.instanceSettings.hostId, + errorMsg: error.message, + errorStack: error.stack ?? '', + }; + + await job.progress(msg); + + ErrorReporterProxy.error(error, { executionId }); + + throw error; + } + @OnShutdown(HIGHEST_SHUTDOWN_PRIORITY) async stop() { - await this.queue.pause(true, true); + const { instanceType } = this.instanceSettings; - this.logger.debug('Queue paused'); + if (instanceType === 'main') await this.stopMain(); + else if (instanceType === 'worker') await this.stopWorker(); + } - this.stopQueueRecovery(); - this.stopQueueMetrics(); + private async stopMain() { + if (this.orchestrationService.isSingleMainSetup) { + await this.queue.pause(true, true); // no more jobs will be picked up + this.logger.debug('Queue paused'); + } - this.logger.debug('Queue recovery and metrics stopped'); + if (this.queueRecoveryContext.timeout) this.stopQueueRecovery(); + if (this.isQueueMetricsEnabled) this.stopQueueMetrics(); + } + private async stopWorker() { let count = 0; while (this.getRunningJobsCount() !== 0) { @@ -161,7 +187,10 @@ export class ScalingService { const job = await this.queue.add(JOB_TYPE_NAME, jobData, jobOptions); - this.logger.info(`Added job ${job.id} (execution ${jobData.executionId})`); + const { executionId } = jobData; + const jobId = job.id; + + this.logger.info(`Enqueued execution ${executionId} (job ${jobId})`, { executionId, jobId }); return job; } @@ -218,7 +247,7 @@ export class ScalingService { */ private registerWorkerListeners() { this.queue.on('global:progress', (jobId: JobId, msg: unknown) => { - if (!this.isPubSubMessage(msg)) return; + if (!this.isJobMessage(msg)) return; if (msg.kind === 'abort-job') this.jobProcessor.stopJob(jobId); }); @@ -258,12 +287,42 @@ export class ScalingService { throw error; }); - this.queue.on('global:progress', (_jobId: JobId, msg: unknown) => { - if (!this.isPubSubMessage(msg)) return; + this.queue.on('global:progress', (jobId: JobId, msg: unknown) => { + if (!this.isJobMessage(msg)) return; - if (msg.kind === 'respond-to-webhook') { - const decodedResponse = this.decodeWebhookResponse(msg.response); - this.activeExecutions.resolveResponsePromise(msg.executionId, decodedResponse); + // completion and failure are reported via `global:progress` to convey more details + // than natively provided by Bull in `global:completed` and `global:failed` events + + switch (msg.kind) { + case 'respond-to-webhook': + const decodedResponse = this.decodeWebhookResponse(msg.response); + this.activeExecutions.resolveResponsePromise(msg.executionId, decodedResponse); + break; + case 'job-finished': + this.logger.info(`Execution ${msg.executionId} (job ${jobId}) finished successfully`, { + workerId: msg.workerId, + executionId: msg.executionId, + jobId, + }); + break; + case 'job-failed': + this.logger.error( + [ + `Execution ${msg.executionId} (job ${jobId}) failed`, + msg.errorStack ? `\n${msg.errorStack}\n` : '', + ].join(''), + { + workerId: msg.workerId, + errorMsg: msg.errorMsg, + executionId: msg.executionId, + jobId, + }, + ); + break; + case 'abort-job': + break; // only for worker + default: + assertNever(msg); } }); @@ -273,7 +332,8 @@ export class ScalingService { } } - private isPubSubMessage(candidate: unknown): candidate is JobReport { + /** Whether the argument is a message sent via Bull's internal pubsub setup. */ + private isJobMessage(candidate: unknown): candidate is JobMessage { return typeof candidate === 'object' && candidate !== null && 'kind' in candidate; } @@ -345,6 +405,8 @@ export class ScalingService { if (this.queueMetricsInterval) { clearInterval(this.queueMetricsInterval); this.queueMetricsInterval = undefined; + + this.logger.debug('Queue metrics collection stopped'); } } @@ -379,6 +441,8 @@ export class ScalingService { private stopQueueRecovery() { clearTimeout(this.queueRecoveryContext.timeout); + + this.logger.debug('Queue recovery stopped'); } /** diff --git a/packages/cli/src/scaling/scaling.types.ts b/packages/cli/src/scaling/scaling.types.ts index fa8210450f..ae7e790a16 100644 --- a/packages/cli/src/scaling/scaling.types.ts +++ b/packages/cli/src/scaling/scaling.types.ts @@ -23,19 +23,44 @@ export type JobStatus = Bull.JobStatus; export type JobOptions = Bull.JobOptions; -export type JobReport = JobReportToMain | JobReportToWorker; +/** + * Message sent by main to worker and vice versa about a job. `JobMessage` is + * sent via Bull's internal pubsub setup - do not confuse with `PubSub.Command` + * and `PubSub.Response`, which are sent via n8n's own pubsub setup to keep + * main and worker processes in sync outside of a job's lifecycle. + */ +export type JobMessage = + | RespondToWebhookMessage + | JobFinishedMessage + | JobFailedMessage + | AbortJobMessage; -type JobReportToMain = RespondToWebhookMessage; - -type JobReportToWorker = AbortJobMessage; - -type RespondToWebhookMessage = { +/** Message sent by worker to main to respond to a webhook. */ +export type RespondToWebhookMessage = { kind: 'respond-to-webhook'; executionId: string; response: IExecuteResponsePromiseData; + workerId: string; }; -type AbortJobMessage = { +/** Message sent by worker to main to report a job has finished successfully. */ +export type JobFinishedMessage = { + kind: 'job-finished'; + executionId: string; + workerId: string; +}; + +/** Message sent by worker to main to report a job has failed. */ +export type JobFailedMessage = { + kind: 'job-failed'; + executionId: string; + workerId: string; + errorMsg: string; + errorStack: string; +}; + +/** Message sent by main to worker to abort a job. */ +export type AbortJobMessage = { kind: 'abort-job'; }; diff --git a/packages/cli/src/scaling/worker-server.ts b/packages/cli/src/scaling/worker-server.ts index 3cf6995882..ee622d789c 100644 --- a/packages/cli/src/scaling/worker-server.ts +++ b/packages/cli/src/scaling/worker-server.ts @@ -58,6 +58,8 @@ export class WorkerServer { ) { assert(this.instanceSettings.instanceType === 'worker'); + this.logger = this.logger.scoped('scaling'); + this.app = express(); this.app.disable('x-powered-by'); @@ -84,6 +86,10 @@ export class WorkerServer { await this.mountEndpoints(); + this.logger.debug('Worker server initialized', { + endpoints: Object.keys(this.endpointsConfig), + }); + await new Promise((resolve) => this.server.listen(this.port, this.address, resolve)); await this.externalHooks.run('worker.ready'); @@ -141,6 +147,8 @@ export class WorkerServer { this.overwritesLoaded = true; + this.logger.debug('Worker loaded credentials overwrites'); + ResponseHelper.sendSuccessResponse(res, { success: true }, true, 200); } } diff --git a/packages/cli/src/scaling/worker-status.service.ts b/packages/cli/src/scaling/worker-status.service.ts index 725cbb0ca7..a50a1b8d2e 100644 --- a/packages/cli/src/scaling/worker-status.service.ts +++ b/packages/cli/src/scaling/worker-status.service.ts @@ -1,19 +1,22 @@ import type { WorkerStatus } from '@n8n/api-types'; +import { InstanceSettings } from 'n8n-core'; import os from 'node:os'; import { Service } from 'typedi'; -import config from '@/config'; import { N8N_VERSION } from '@/constants'; import { JobProcessor } from './job-processor'; @Service() export class WorkerStatusService { - constructor(private readonly jobProcessor: JobProcessor) {} + constructor( + private readonly jobProcessor: JobProcessor, + private readonly instanceSettings: InstanceSettings, + ) {} generateStatus(): WorkerStatus { return { - senderId: config.getEnv('redis.queueModeId'), + senderId: this.instanceSettings.hostId, runningJobsSummary: this.jobProcessor.getRunningJobsSummary(), freeMem: os.freemem(), totalMem: os.totalmem(), diff --git a/packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts b/packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts index ab7873e808..0c8d84211e 100644 --- a/packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts +++ b/packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts @@ -1,7 +1,7 @@ +import { SecurityConfig } from '@n8n/config'; import type { IWorkflowBase } from 'n8n-workflow'; import { Service } from 'typedi'; -import config from '@/config'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import { ExecutionDataRepository } from '@/databases/repositories/execution-data.repository'; @@ -15,10 +15,11 @@ export class CredentialsRiskReporter implements RiskReporter { private readonly credentialsRepository: CredentialsRepository, private readonly executionRepository: ExecutionRepository, private readonly executionDataRepository: ExecutionDataRepository, + private readonly securityConfig: SecurityConfig, ) {} async report(workflows: WorkflowEntity[]) { - const days = config.getEnv('security.audit.daysAbandonedWorkflow'); + const days = this.securityConfig.daysAbandonedWorkflow; const allExistingCreds = await this.getAllExistingCreds(); const { credsInAnyUse, credsInActiveUse } = await this.getAllCredsInUse(workflows); diff --git a/packages/cli/src/security-audit/security-audit.service.ts b/packages/cli/src/security-audit/security-audit.service.ts index 19582450c4..97b5424a19 100644 --- a/packages/cli/src/security-audit/security-audit.service.ts +++ b/packages/cli/src/security-audit/security-audit.service.ts @@ -1,3 +1,4 @@ +import { SecurityConfig } from '@n8n/config'; import Container, { Service } from 'typedi'; import config from '@/config'; @@ -8,7 +9,10 @@ import { toReportTitle } from '@/security-audit/utils'; @Service() export class SecurityAuditService { - constructor(private readonly workflowRepository: WorkflowRepository) {} + constructor( + private readonly workflowRepository: WorkflowRepository, + private readonly securityConfig: SecurityConfig, + ) {} private reporters: { [name: string]: RiskReporter; @@ -19,7 +23,7 @@ export class SecurityAuditService { await this.initReporters(categories); - const daysFromEnv = config.getEnv('security.audit.daysAbandonedWorkflow'); + const daysFromEnv = this.securityConfig.daysAbandonedWorkflow; if (daysAbandonedWorkflow) { config.set('security.audit.daysAbandonedWorkflow', daysAbandonedWorkflow); diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 00971d71a5..3cfd93054b 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -79,8 +79,9 @@ export class Server extends AbstractServer { private readonly orchestrationService: OrchestrationService, private readonly postHogClient: PostHogClient, private readonly eventService: EventService, + private readonly instanceSettings: InstanceSettings, ) { - super('main'); + super(); this.testWebhooksEnabled = true; this.webhooksEnabled = !this.globalConfig.endpoints.disableProductionWebhooksOnMainProcess; @@ -97,7 +98,7 @@ export class Server extends AbstractServer { this.endpointPresetCredentials = this.globalConfig.credentials.overwrite.endpoint; await super.start(); - this.logger.debug(`Server ID: ${this.uniqueInstanceId}`); + this.logger.debug(`Server ID: ${this.instanceSettings.hostId}`); if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') { void this.loadNodesAndCredentials.setupHotReload(); diff --git a/packages/cli/src/services/__tests__/orchestration.service.test.ts b/packages/cli/src/services/__tests__/orchestration.service.test.ts index 6c66573047..a8e72c49bf 100644 --- a/packages/cli/src/services/__tests__/orchestration.service.test.ts +++ b/packages/cli/src/services/__tests__/orchestration.service.test.ts @@ -1,7 +1,6 @@ import type Redis from 'ioredis'; import { mock } from 'jest-mock-extended'; import { InstanceSettings } from 'n8n-core'; -import type { WorkflowActivateMode } from 'n8n-workflow'; import Container from 'typedi'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; @@ -23,15 +22,11 @@ redisClientService.createClient.mockReturnValue(mockRedisClient); const os = Container.get(OrchestrationService); mockInstance(ActiveWorkflowManager); -let queueModeId: string; - describe('Orchestration Service', () => { mockInstance(Push); mockInstance(ExternalSecretsManager); beforeAll(async () => { - queueModeId = config.get('redis.queueModeId'); - // @ts-expect-error readonly property instanceSettings.instanceType = 'main'; }); @@ -48,37 +43,5 @@ describe('Orchestration Service', () => { await os.init(); // @ts-expect-error Private field expect(os.publisher).toBeDefined(); - expect(queueModeId).toBeDefined(); - }); - - describe('shouldAddWebhooks', () => { - test('should return true for init', () => { - // We want to ensure that webhooks are populated on init - // more https://github.com/n8n-io/n8n/pull/8830 - const result = os.shouldAddWebhooks('init'); - expect(result).toBe(true); - }); - - test('should return false for leadershipChange', () => { - const result = os.shouldAddWebhooks('leadershipChange'); - expect(result).toBe(false); - }); - - test('should return true for update or activate when is leader', () => { - const modes = ['update', 'activate'] as WorkflowActivateMode[]; - for (const mode of modes) { - const result = os.shouldAddWebhooks(mode); - expect(result).toBe(true); - } - }); - - test('should return false for update or activate when not leader', () => { - instanceSettings.markAsFollower(); - const modes = ['update', 'activate'] as WorkflowActivateMode[]; - for (const mode of modes) { - const result = os.shouldAddWebhooks(mode); - expect(result).toBe(false); - } - }); }); }); diff --git a/packages/cli/src/services/__tests__/public-api-key.service.test.ts b/packages/cli/src/services/__tests__/public-api-key.service.test.ts new file mode 100644 index 0000000000..7c60b62983 --- /dev/null +++ b/packages/cli/src/services/__tests__/public-api-key.service.test.ts @@ -0,0 +1,147 @@ +import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; +import type { OpenAPIV3 } from 'openapi-types'; + +import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; +import { UserRepository } from '@/databases/repositories/user.repository'; +import { getConnection } from '@/db'; +import type { EventService } from '@/events/event.service'; +import type { AuthenticatedRequest } from '@/requests'; +import { createOwnerWithApiKey } from '@test-integration/db/users'; +import * as testDb from '@test-integration/test-db'; + +import { JwtService } from '../jwt.service'; +import { PublicApiKeyService } from '../public-api-key.service'; + +const mockReqWith = (apiKey: string, path: string, method: string) => { + return mock({ + path, + method, + headers: { + 'x-n8n-api-key': apiKey, + }, + }); +}; + +const instanceSettings = mock({ encryptionKey: 'test-key' }); + +const eventService = mock(); + +const securitySchema = mock({ + name: 'X-N8N-API-KEY', +}); + +const jwtService = new JwtService(instanceSettings); + +let userRepository: UserRepository; +let apiKeyRepository: ApiKeyRepository; + +describe('PublicApiKeyService', () => { + beforeEach(async () => { + await testDb.truncate(['User']); + jest.clearAllMocks(); + }); + + beforeAll(async () => { + await testDb.init(); + userRepository = new UserRepository(getConnection()); + apiKeyRepository = new ApiKeyRepository(getConnection()); + }); + + afterAll(async () => { + await testDb.terminate(); + }); + + describe('getAuthMiddleware', () => { + it('should return false if api key is invalid', async () => { + //Arrange + + const apiKey = 'invalid'; + const path = '/test'; + const method = 'GET'; + const apiVersion = 'v1'; + + const publicApiKeyService = new PublicApiKeyService( + apiKeyRepository, + userRepository, + jwtService, + eventService, + ); + + const middleware = publicApiKeyService.getAuthMiddleware(apiVersion); + + //Act + + const response = await middleware(mockReqWith(apiKey, path, method), {}, securitySchema); + + //Assert + + expect(response).toBe(false); + }); + + it('should return false if valid api key is not in database', async () => { + //Arrange + + const apiKey = jwtService.sign({ sub: '123' }); + const path = '/test'; + const method = 'GET'; + const apiVersion = 'v1'; + + const publicApiKeyService = new PublicApiKeyService( + apiKeyRepository, + userRepository, + jwtService, + eventService, + ); + + const middleware = publicApiKeyService.getAuthMiddleware(apiVersion); + + //Act + + const response = await middleware(mockReqWith(apiKey, path, method), {}, securitySchema); + + //Assert + + expect(response).toBe(false); + }); + + it('should return true if valid api key exist in the database', async () => { + //Arrange + + const path = '/test'; + const method = 'GET'; + const apiVersion = 'v1'; + + const publicApiKeyService = new PublicApiKeyService( + apiKeyRepository, + userRepository, + jwtService, + eventService, + ); + + const owner = await createOwnerWithApiKey(); + + const [{ apiKey }] = owner.apiKeys; + + const middleware = publicApiKeyService.getAuthMiddleware(apiVersion); + + //Act + + const response = await middleware(mockReqWith(apiKey, path, method), {}, securitySchema); + + //Assert + + expect(response).toBe(true); + expect(eventService.emit).toHaveBeenCalledTimes(1); + expect(eventService.emit).toHaveBeenCalledWith( + 'public-api-invoked', + expect.objectContaining({ + userId: owner.id, + path, + method, + apiVersion: 'v1', + }), + ); + }); + }); +}); diff --git a/packages/cli/src/services/community-packages.service.ts b/packages/cli/src/services/community-packages.service.ts index b157119cf2..4906a6ef33 100644 --- a/packages/cli/src/services/community-packages.service.ts +++ b/packages/cli/src/services/community-packages.service.ts @@ -23,10 +23,9 @@ import type { CommunityPackages } from '@/interfaces'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { Logger } from '@/logging/logger.service'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { toError } from '@/utils'; -import { OrchestrationService } from './orchestration.service'; - const DEFAULT_REGISTRY = 'https://registry.npmjs.org'; const { @@ -60,7 +59,7 @@ export class CommunityPackagesService { private readonly logger: Logger, private readonly installedPackageRepository: InstalledPackagesRepository, private readonly loadNodesAndCredentials: LoadNodesAndCredentials, - private readonly orchestrationService: OrchestrationService, + private readonly publisher: Publisher, private readonly license: License, private readonly globalConfig: GlobalConfig, ) {} @@ -322,7 +321,10 @@ export class CommunityPackagesService { async removePackage(packageName: string, installedPackage: InstalledPackages): Promise { await this.removeNpmPackage(packageName); await this.removePackageFromDatabase(installedPackage); - await this.orchestrationService.publish('community-package-uninstall', { packageName }); + void this.publisher.publishCommand({ + command: 'community-package-uninstall', + payload: { packageName }, + }); } private getNpmRegistry() { @@ -368,10 +370,10 @@ export class CommunityPackagesService { await this.removePackageFromDatabase(options.installedPackage); } const installedPackage = await this.persistInstalledPackage(loader); - await this.orchestrationService.publish( - isUpdate ? 'community-package-update' : 'community-package-install', - { packageName, packageVersion }, - ); + void this.publisher.publishCommand({ + command: isUpdate ? 'community-package-update' : 'community-package-install', + payload: { packageName, packageVersion }, + }); await this.loadNodesAndCredentials.postProcessLoaders(); this.logger.info(`Community package installed: ${packageName}`); return installedPackage; diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index a83158e96e..6cad4a4f24 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -1,5 +1,5 @@ import type { FrontendSettings, ITelemetrySettings } from '@n8n/api-types'; -import { GlobalConfig } from '@n8n/config'; +import { GlobalConfig, FrontendConfig, SecurityConfig } from '@n8n/config'; import { createWriteStream } from 'fs'; import { mkdir } from 'fs/promises'; import uniq from 'lodash/uniq'; @@ -46,6 +46,8 @@ export class FrontendService { private readonly mailer: UserManagementMailer, private readonly instanceSettings: InstanceSettings, private readonly urlService: UrlService, + private readonly securityConfig: SecurityConfig, + private readonly frontendConfig: FrontendConfig, ) { loadNodesAndCredentials.addPostProcessor(async () => await this.generateTypes()); void this.generateTypes(); @@ -96,7 +98,7 @@ export class FrontendService { executionTimeout: config.getEnv('executions.timeout'), maxExecutionTimeout: config.getEnv('executions.maxTimeout'), workflowCallerPolicyDefaultOption: this.globalConfig.workflows.callerPolicyDefaultOption, - timezone: config.getEnv('generic.timezone'), + timezone: this.globalConfig.generic.timezone, urlBaseWebhook: this.urlService.getWebhookBaseUrl(), urlBaseEditor: instanceBaseUrl, binaryDataMode: config.getEnv('binaryDataManager.mode'), @@ -106,7 +108,7 @@ export class FrontendService { authCookie: { secure: config.getEnv('secure_cookie'), }, - releaseChannel: config.getEnv('generic.releaseChannel'), + releaseChannel: this.globalConfig.generic.releaseChannel, oauthCallbackUrls: { oauth1: `${instanceBaseUrl}/${restEndpoint}/oauth1-credential/callback`, oauth2: `${instanceBaseUrl}/${restEndpoint}/oauth2-credential/callback`, @@ -201,7 +203,7 @@ export class FrontendService { hideUsagePage: config.getEnv('hideUsagePage'), license: { consumerId: 'unknown', - environment: config.getEnv('license.tenantId') === 1 ? 'production' : 'staging', + environment: this.globalConfig.license.tenantId === 1 ? 'production' : 'staging', }, variables: { limit: 0, @@ -225,8 +227,9 @@ export class FrontendService { maxCount: config.getEnv('executions.pruneDataMaxCount'), }, security: { - blockFileAccessToN8nFiles: config.getEnv('security.blockFileAccessToN8nFiles'), + blockFileAccessToN8nFiles: this.securityConfig.blockFileAccessToN8nFiles, }, + betaFeatures: this.frontendConfig.betaFeatures, }; } diff --git a/packages/cli/src/services/orchestration.handler.base.service.ts b/packages/cli/src/services/orchestration.handler.base.service.ts deleted file mode 100644 index e994ff6308..0000000000 --- a/packages/cli/src/services/orchestration.handler.base.service.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { MainResponseReceivedHandlerOptions } from './orchestration/main/types'; -import type { WorkerCommandReceivedHandlerOptions } from './orchestration/worker/types'; - -export abstract class OrchestrationHandlerService { - protected initialized = false; - - async init() { - await this.initSubscriber(); - this.initialized = true; - } - - async initWithOptions( - options: WorkerCommandReceivedHandlerOptions | MainResponseReceivedHandlerOptions, - ) { - await this.initSubscriber(options); - this.initialized = true; - } - - async shutdown() { - this.initialized = false; - } - - protected abstract initSubscriber( - options?: WorkerCommandReceivedHandlerOptions | MainResponseReceivedHandlerOptions, - ): Promise; -} diff --git a/packages/cli/src/services/orchestration.service.ts b/packages/cli/src/services/orchestration.service.ts index b8aba46285..225badbf18 100644 --- a/packages/cli/src/services/orchestration.service.ts +++ b/packages/cli/src/services/orchestration.service.ts @@ -1,21 +1,19 @@ +import { GlobalConfig } from '@n8n/config'; import { InstanceSettings } from 'n8n-core'; -import type { WorkflowActivateMode } from 'n8n-workflow'; import Container, { Service } from 'typedi'; import config from '@/config'; -import type { PubSubCommandMap } from '@/events/maps/pub-sub.event-map'; -import { Logger } from '@/logging/logger.service'; import type { Publisher } from '@/scaling/pubsub/publisher.service'; import type { Subscriber } from '@/scaling/pubsub/subscriber.service'; -import { MultiMainSetup } from './orchestration/main/multi-main-setup.ee'; +import { MultiMainSetup } from '../scaling/multi-main-setup.ee'; @Service() export class OrchestrationService { constructor( - private readonly logger: Logger, readonly instanceSettings: InstanceSettings, readonly multiMainSetup: MultiMainSetup, + readonly globalConfig: GlobalConfig, ) {} private publisher: Publisher; @@ -33,7 +31,7 @@ export class OrchestrationService { get isMultiMainSetupEnabled() { return ( config.getEnv('executions.mode') === 'queue' && - config.getEnv('multiMainSetup.enabled') && + this.globalConfig.multiMainSetup.enabled && this.instanceSettings.instanceType === 'main' && this.isMultiMainSetupLicensed ); @@ -43,20 +41,6 @@ export class OrchestrationService { return !this.isMultiMainSetupEnabled; } - get instanceId() { - return config.getEnv('redis.queueModeId'); - } - - /** @deprecated use InstanceSettings.isLeader */ - get isLeader() { - return this.instanceSettings.isLeader; - } - - /** @deprecated use InstanceSettings.isFollower */ - get isFollower() { - return this.instanceSettings.isFollower; - } - sanityCheck() { return this.isInitialized && config.get('executions.mode') === 'queue'; } @@ -92,68 +76,4 @@ export class OrchestrationService { this.isInitialized = false; } - - // ---------------------------------- - // pubsub - // ---------------------------------- - - async publish( - commandKey: CommandKey, - payload?: PubSubCommandMap[CommandKey], - ) { - if (!this.sanityCheck()) return; - - this.logger.debug( - `[Instance ID ${this.instanceId}] Publishing command "${commandKey}"`, - payload, - ); - - await this.publisher.publishCommand({ command: commandKey, payload }); - } - - // ---------------------------------- - // workers status - // ---------------------------------- - - async getWorkerStatus(id?: string) { - if (!this.sanityCheck()) return; - - const command = 'get-worker-status'; - - this.logger.debug(`Sending "${command}" to command channel`); - - await this.publisher.publishCommand({ - command, - targets: id ? [id] : undefined, - }); - } - - // ---------------------------------- - // activations - // ---------------------------------- - - /** - * Whether this instance may add webhooks to the `webhook_entity` table. - */ - shouldAddWebhooks(activationMode: WorkflowActivateMode) { - // Always try to populate the webhook entity table as well as register the webhooks - // to prevent issues with users upgrading from a version < 1.15, where the webhook entity - // was cleared on shutdown to anything past 1.28.0, where we stopped populating it on init, - // causing all webhooks to break - if (activationMode === 'init') return true; - - if (activationMode === 'leadershipChange') return false; - - return this.isLeader; // 'update' or 'activate' - } - - /** - * Whether this instance may add triggers and pollers to memory. - * - * In both single- and multi-main setup, only the leader is allowed to manage - * triggers and pollers in memory, to ensure they are not duplicated. - */ - shouldAddTriggersAndPollers() { - return this.isLeader; - } } diff --git a/packages/cli/src/services/orchestration/main/types.ts b/packages/cli/src/services/orchestration/main/types.ts deleted file mode 100644 index 7388a55032..0000000000 --- a/packages/cli/src/services/orchestration/main/types.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { Publisher } from '@/scaling/pubsub/publisher.service'; - -export type MainResponseReceivedHandlerOptions = { - queueModeId: string; - publisher: Publisher; -}; diff --git a/packages/cli/src/services/orchestration/webhook/orchestration.webhook.service.ts b/packages/cli/src/services/orchestration/webhook/orchestration.webhook.service.ts deleted file mode 100644 index 6b1c86fc6a..0000000000 --- a/packages/cli/src/services/orchestration/webhook/orchestration.webhook.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Service } from 'typedi'; - -import config from '@/config'; - -import { OrchestrationService } from '../../orchestration.service'; - -@Service() -export class OrchestrationWebhookService extends OrchestrationService { - sanityCheck(): boolean { - return ( - this.isInitialized && - config.get('executions.mode') === 'queue' && - this.instanceSettings.instanceType === 'webhook' - ); - } -} diff --git a/packages/cli/src/services/orchestration/worker/orchestration.worker.service.ts b/packages/cli/src/services/orchestration/worker/orchestration.worker.service.ts deleted file mode 100644 index 1d0d822aeb..0000000000 --- a/packages/cli/src/services/orchestration/worker/orchestration.worker.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Service } from 'typedi'; - -import config from '@/config'; - -import { OrchestrationService } from '../../orchestration.service'; - -@Service() -export class OrchestrationWorkerService extends OrchestrationService { - sanityCheck(): boolean { - return ( - this.isInitialized && - config.get('executions.mode') === 'queue' && - this.instanceSettings.instanceType === 'worker' - ); - } -} diff --git a/packages/cli/src/services/orchestration/worker/types.ts b/packages/cli/src/services/orchestration/worker/types.ts deleted file mode 100644 index d821a194b2..0000000000 --- a/packages/cli/src/services/orchestration/worker/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { RunningJobSummary } from '@n8n/api-types'; - -import type { Publisher } from '@/scaling/pubsub/publisher.service'; - -export interface WorkerCommandReceivedHandlerOptions { - queueModeId: string; - publisher: Publisher; - getRunningJobIds: () => Array; - getRunningJobsSummary: () => RunningJobSummary[]; -} diff --git a/packages/cli/src/services/pruning.service.ts b/packages/cli/src/services/pruning.service.ts index 48d4b0db3b..0859dddd39 100644 --- a/packages/cli/src/services/pruning.service.ts +++ b/packages/cli/src/services/pruning.service.ts @@ -1,3 +1,4 @@ +import { GlobalConfig } from '@n8n/config'; import { BinaryDataService, InstanceSettings } from 'n8n-core'; import { jsonStringify } from 'n8n-workflow'; import { Service } from 'typedi'; @@ -31,13 +32,15 @@ export class PruningService { private readonly executionRepository: ExecutionRepository, private readonly binaryDataService: BinaryDataService, private readonly orchestrationService: OrchestrationService, + private readonly globalConfig: GlobalConfig, ) {} /** * @important Requires `OrchestrationService` to be initialized. */ init() { - const { isLeader, isMultiMainSetupEnabled } = this.orchestrationService; + const { isLeader } = this.instanceSettings; + const { isMultiMainSetupEnabled } = this.orchestrationService; if (isLeader) this.startPruning(); @@ -53,7 +56,7 @@ export class PruningService { return false; } - if (config.getEnv('multiMainSetup.enabled') && instanceType === 'main' && isFollower) { + if (this.globalConfig.multiMainSetup.enabled && instanceType === 'main' && isFollower) { return false; } diff --git a/packages/cli/src/services/public-api-key.service.ts b/packages/cli/src/services/public-api-key.service.ts index e689f3c019..bca3cd0d62 100644 --- a/packages/cli/src/services/public-api-key.service.ts +++ b/packages/cli/src/services/public-api-key.service.ts @@ -1,16 +1,28 @@ -import { randomBytes } from 'node:crypto'; -import Container, { Service } from 'typedi'; +import type { OpenAPIV3 } from 'openapi-types'; +import { Service } from 'typedi'; import { ApiKey } from '@/databases/entities/api-key'; import type { User } from '@/databases/entities/user'; import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; import { UserRepository } from '@/databases/repositories/user.repository'; +import { EventService } from '@/events/event.service'; +import type { AuthenticatedRequest } from '@/requests'; -export const API_KEY_PREFIX = 'n8n_api_'; +import { JwtService } from './jwt.service'; + +const API_KEY_AUDIENCE = 'public-api'; +const API_KEY_ISSUER = 'n8n'; +const REDACT_API_KEY_REVEAL_COUNT = 15; +const REDACT_API_KEY_MAX_LENGTH = 80; @Service() export class PublicApiKeyService { - constructor(private readonly apiKeyRepository: ApiKeyRepository) {} + constructor( + private readonly apiKeyRepository: ApiKeyRepository, + private readonly userRepository: UserRepository, + private readonly jwtService: JwtService, + private readonly eventService: EventService, + ) {} /** * Creates a new public API key for the specified user. @@ -18,7 +30,7 @@ export class PublicApiKeyService { * @returns A promise that resolves to the newly created API key. */ async createPublicApiKeyForUser(user: User) { - const apiKey = this.createApiKeyString(); + const apiKey = this.generateApiKey(user); await this.apiKeyRepository.upsert( this.apiKeyRepository.create({ userId: user.id, @@ -48,8 +60,8 @@ export class PublicApiKeyService { await this.apiKeyRepository.delete({ userId: user.id, id: apiKeyId }); } - async getUserForApiKey(apiKey: string) { - return await Container.get(UserRepository) + private async getUserForApiKey(apiKey: string) { + return await this.userRepository .createQueryBuilder('user') .innerJoin(ApiKey, 'apiKey', 'apiKey.userId = user.id') .where('apiKey.apiKey = :apiKey', { apiKey }) @@ -68,13 +80,39 @@ export class PublicApiKeyService { * ``` */ redactApiKey(apiKey: string) { - const keepLength = 5; - return ( - API_KEY_PREFIX + - apiKey.slice(API_KEY_PREFIX.length, API_KEY_PREFIX.length + keepLength) + - '*'.repeat(apiKey.length - API_KEY_PREFIX.length - keepLength) - ); + const visiblePart = apiKey.slice(0, REDACT_API_KEY_REVEAL_COUNT); + const redactedPart = '*'.repeat(apiKey.length - REDACT_API_KEY_REVEAL_COUNT); + + const completeRedactedApiKey = visiblePart + redactedPart; + + return completeRedactedApiKey.slice(0, REDACT_API_KEY_MAX_LENGTH); } - createApiKeyString = () => `${API_KEY_PREFIX}${randomBytes(40).toString('hex')}`; + getAuthMiddleware(version: string) { + return async ( + req: AuthenticatedRequest, + _scopes: unknown, + schema: OpenAPIV3.ApiKeySecurityScheme, + ): Promise => { + const providedApiKey = req.headers[schema.name.toLowerCase()] as string; + + const user = await this.getUserForApiKey(providedApiKey); + + if (!user) return false; + + this.eventService.emit('public-api-invoked', { + userId: user.id, + path: req.path, + method: req.method, + apiVersion: version, + }); + + req.user = user; + + return true; + }; + } + + private generateApiKey = (user: User) => + this.jwtService.sign({ sub: user.id, iss: API_KEY_ISSUER, aud: API_KEY_AUDIENCE }); } diff --git a/packages/cli/src/services/redis-client.service.ts b/packages/cli/src/services/redis-client.service.ts index 5eaa6edc1d..c584530165 100644 --- a/packages/cli/src/services/redis-client.service.ts +++ b/packages/cli/src/services/redis-client.service.ts @@ -37,6 +37,9 @@ export class RedisClientService extends TypedEmitter { private readonly globalConfig: GlobalConfig, ) { super(); + + this.logger = this.logger.scoped(['redis', 'scaling']); + this.registerListeners(); } @@ -99,9 +102,11 @@ export class RedisClientService extends TypedEmitter { options.host = host; options.port = port; - this.logger.debug('[Redis] Initializing regular client', { type, host, port }); + const client = new ioRedis(options); - return new ioRedis(options); + this.logger.debug(`Started Redis client ${type}`, { type, host, port }); + + return client; } private createClusterClient({ @@ -115,12 +120,14 @@ export class RedisClientService extends TypedEmitter { const clusterNodes = this.clusterNodes(); - this.logger.debug('[Redis] Initializing cluster client', { type, clusterNodes }); - - return new ioRedis.Cluster(clusterNodes, { + const clusterClient = new ioRedis.Cluster(clusterNodes, { redisOptions: options, clusterRetryStrategy: this.retryStrategy(), }); + + this.logger.debug(`Started Redis cluster client ${type}`, { type, clusterNodes }); + + return clusterClient; } private getOptions({ extraOptions }: { extraOptions?: RedisOptions }) { diff --git a/packages/cli/src/services/user.service.ts b/packages/cli/src/services/user.service.ts index 1668878a8c..ba02375aba 100644 --- a/packages/cli/src/services/user.service.ts +++ b/packages/cli/src/services/user.service.ts @@ -130,6 +130,7 @@ export class UserService { email, inviteAcceptUrl, emailSent: false, + role, }, error: '', }; diff --git a/packages/cli/src/sso/saml/routes/__tests__/saml.controller.ee.test.ts b/packages/cli/src/sso/saml/routes/__tests__/saml.controller.ee.test.ts new file mode 100644 index 0000000000..c4a33ed441 --- /dev/null +++ b/packages/cli/src/sso/saml/routes/__tests__/saml.controller.ee.test.ts @@ -0,0 +1,77 @@ +import { type Response } from 'express'; +import { mock } from 'jest-mock-extended'; + +import type { User } from '@/databases/entities/user'; +import { UrlService } from '@/services/url.service'; +import { mockInstance } from '@test/mocking'; + +import { SamlService } from '../../saml.service.ee'; +import { getServiceProviderConfigTestReturnUrl } from '../../service-provider.ee'; +import type { SamlConfiguration } from '../../types/requests'; +import type { SamlUserAttributes } from '../../types/saml-user-attributes'; +import { SamlController } from '../saml.controller.ee'; + +const urlService = mockInstance(UrlService); +urlService.getInstanceBaseUrl.mockReturnValue(''); +const samlService = mockInstance(SamlService); +const controller = new SamlController(mock(), samlService, mock(), mock()); + +const user = mock({ + id: '123', + password: 'password', + authIdentities: [], + role: 'global:owner', +}); + +const attributes: SamlUserAttributes = { + email: 'test@example.com', + firstName: 'Test', + lastName: 'User', + userPrincipalName: 'upn:test@example.com', +}; + +describe('Test views', () => { + test('Should render success with template', async () => { + const req = mock(); + const res = mock(); + + req.body.RelayState = getServiceProviderConfigTestReturnUrl(); + samlService.handleSamlLogin.mockResolvedValueOnce({ + authenticatedUser: user, + attributes, + onboardingRequired: false, + }); + + await controller.acsPost(req, res); + + expect(res.render).toBeCalledWith('saml-connection-test-success', attributes); + }); + + test('Should render failure with template', async () => { + const req = mock(); + const res = mock(); + + req.body.RelayState = getServiceProviderConfigTestReturnUrl(); + samlService.handleSamlLogin.mockResolvedValueOnce({ + authenticatedUser: undefined, + attributes, + onboardingRequired: false, + }); + + await controller.acsPost(req, res); + + expect(res.render).toBeCalledWith('saml-connection-test-failed', { message: '', attributes }); + }); + + test('Should render error with template', async () => { + const req = mock(); + const res = mock(); + + req.body.RelayState = getServiceProviderConfigTestReturnUrl(); + samlService.handleSamlLogin.mockRejectedValueOnce(new Error('Test Error')); + + await controller.acsPost(req, res); + + expect(res.render).toBeCalledWith('saml-connection-test-failed', { message: 'Test Error' }); + }); +}); diff --git a/packages/cli/src/sso/saml/routes/saml.controller.ee.ts b/packages/cli/src/sso/saml/routes/saml.controller.ee.ts index 77c8c19291..c7b954914b 100644 --- a/packages/cli/src/sso/saml/routes/saml.controller.ee.ts +++ b/packages/cli/src/sso/saml/routes/saml.controller.ee.ts @@ -10,6 +10,7 @@ import { AuthError } from '@/errors/response-errors/auth.error'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { EventService } from '@/events/event.service'; import { AuthenticatedRequest } from '@/requests'; +import { sendErrorResponse } from '@/response-helper'; import { UrlService } from '@/services/url.service'; import { @@ -26,8 +27,6 @@ import { import type { SamlLoginBinding } from '../types'; import { SamlConfiguration } from '../types/requests'; import { getInitSSOFormView } from '../views/init-sso-post'; -import { getSamlConnectionTestFailedView } from '../views/saml-connection-test-failed'; -import { getSamlConnectionTestSuccessView } from '../views/saml-connection-test-success'; @RestController('/sso/saml') export class SamlController { @@ -92,7 +91,7 @@ export class SamlController { /** * Assertion Consumer Service endpoint */ - @Get('/acs', { middlewares: [samlLicensedMiddleware], skipAuth: true }) + @Get('/acs', { middlewares: [samlLicensedMiddleware], skipAuth: true, usesTemplates: true }) async acsGet(req: SamlConfiguration.AcsRequest, res: express.Response) { return await this.acsHandler(req, res, 'redirect'); } @@ -100,7 +99,7 @@ export class SamlController { /** * Assertion Consumer Service endpoint */ - @Post('/acs', { middlewares: [samlLicensedMiddleware], skipAuth: true }) + @Post('/acs', { middlewares: [samlLicensedMiddleware], skipAuth: true, usesTemplates: true }) async acsPost(req: SamlConfiguration.AcsRequest, res: express.Response) { return await this.acsHandler(req, res, 'post'); } @@ -120,9 +119,12 @@ export class SamlController { // if RelayState is set to the test connection Url, this is a test connection if (isConnectionTestRequest(req)) { if (loginResult.authenticatedUser) { - return res.send(getSamlConnectionTestSuccessView(loginResult.attributes)); + return res.render('saml-connection-test-success', loginResult.attributes); } else { - return res.send(getSamlConnectionTestFailedView('', loginResult.attributes)); + return res.render('saml-connection-test-failed', { + message: '', + attributes: loginResult.attributes, + }); } } if (loginResult.authenticatedUser) { @@ -148,16 +150,21 @@ export class SamlController { userEmail: loginResult.attributes.email ?? 'unknown', authenticationMethod: 'saml', }); - throw new AuthError('SAML Authentication failed'); + // Need to manually send the error response since we're using templates + return sendErrorResponse(res, new AuthError('SAML Authentication failed')); } catch (error) { if (isConnectionTestRequest(req)) { - return res.send(getSamlConnectionTestFailedView((error as Error).message)); + return res.render('saml-connection-test-failed', { message: (error as Error).message }); } this.eventService.emit('user-login-failed', { userEmail: 'unknown', authenticationMethod: 'saml', }); - throw new AuthError('SAML Authentication failed: ' + (error as Error).message); + // Need to manually send the error response since we're using templates + return sendErrorResponse( + res, + new AuthError('SAML Authentication failed: ' + (error as Error).message), + ); } } diff --git a/packages/cli/src/sso/saml/views/saml-connection-test-failed.ts b/packages/cli/src/sso/saml/views/saml-connection-test-failed.ts deleted file mode 100644 index 4ce2a3e3ac..0000000000 --- a/packages/cli/src/sso/saml/views/saml-connection-test-failed.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { SamlUserAttributes } from '../types/saml-user-attributes'; - -export function getSamlConnectionTestFailedView( - message: string, - attributes?: SamlUserAttributes, -): string { - return ` - - - n8n - SAML Connection Test Result - - - -
-

SAML Connection Test failed

-

${message ?? 'A common issue could be that no email attribute is set'}

- -

- ${ - attributes - ? ` -

Here are the attributes returned by your SAML IdP:

-
    -
  • Email: ${attributes?.email ?? '(n/a)'}
    • -
  • First Name: ${attributes?.firstName ?? '(n/a)'}
    • -
  • Last Name: ${attributes?.lastName ?? '(n/a)'}
    • -
  • UPN: ${attributes?.userPrincipalName ?? '(n/a)'}
    • -
` - : '' - } -
- - - `; -} diff --git a/packages/cli/src/sso/saml/views/saml-connection-test-success.ts b/packages/cli/src/sso/saml/views/saml-connection-test-success.ts deleted file mode 100644 index f647527cd0..0000000000 --- a/packages/cli/src/sso/saml/views/saml-connection-test-success.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { SamlUserAttributes } from '../types/saml-user-attributes'; - -export function getSamlConnectionTestSuccessView(attributes: SamlUserAttributes): string { - return ` - - - n8n - SAML Connection Test Result - - - -
-

SAML Connection Test was successful

- -

-

Here are the attributes returned by your SAML IdP:

-
    -
  • Email: ${attributes.email ?? '(n/a)'}
    • -
  • First Name: ${attributes.firstName ?? '(n/a)'}
    • -
  • Last Name: ${attributes.lastName ?? '(n/a)'}
    • -
  • UPN: ${attributes.userPrincipalName ?? '(n/a)'}
    • -
-
- - - `; -} diff --git a/packages/cli/src/wait-tracker.ts b/packages/cli/src/wait-tracker.ts index e999c30401..868fafa526 100644 --- a/packages/cli/src/wait-tracker.ts +++ b/packages/cli/src/wait-tracker.ts @@ -1,3 +1,4 @@ +import { InstanceSettings } from 'n8n-core'; import { ApplicationError, ErrorReporterProxy as ErrorReporter, @@ -28,8 +29,9 @@ export class WaitTracker { private readonly ownershipService: OwnershipService, private readonly workflowRunner: WorkflowRunner, private readonly orchestrationService: OrchestrationService, + private readonly instanceSettings: InstanceSettings, ) { - this.logger = this.logger.withScope('executions'); + this.logger = this.logger.scoped('waiting-executions'); } has(executionId: string) { @@ -40,7 +42,8 @@ export class WaitTracker { * @important Requires `OrchestrationService` to be initialized. */ init() { - const { isLeader, isMultiMainSetupEnabled } = this.orchestrationService; + const { isLeader } = this.instanceSettings; + const { isMultiMainSetupEnabled } = this.orchestrationService; if (isLeader) this.startTracking(); diff --git a/packages/cli/src/waiting-forms.ts b/packages/cli/src/waiting-forms.ts deleted file mode 100644 index 6f57e1d8fd..0000000000 --- a/packages/cli/src/waiting-forms.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Service } from 'typedi'; - -import type { IExecutionResponse } from '@/interfaces'; -import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; - -@Service() -export class WaitingForms extends WaitingWebhooks { - protected override includeForms = true; - - protected override logReceivedWebhook(method: string, executionId: string) { - this.logger.debug(`Received waiting-form "${method}" for execution "${executionId}"`); - } - - protected disableNode(execution: IExecutionResponse, method?: string) { - if (method === 'POST') { - execution.data.executionData!.nodeExecutionStack[0].node.disabled = true; - } - } -} diff --git a/packages/cli/src/webhooks/__tests__/test-webhooks.test.ts b/packages/cli/src/webhooks/__tests__/test-webhooks.test.ts index d9228bcb0d..3f8972ad9a 100644 --- a/packages/cli/src/webhooks/__tests__/test-webhooks.test.ts +++ b/packages/cli/src/webhooks/__tests__/test-webhooks.test.ts @@ -39,7 +39,7 @@ let testWebhooks: TestWebhooks; describe('TestWebhooks', () => { beforeAll(() => { - testWebhooks = new TestWebhooks(mock(), mock(), registrations, mock()); + testWebhooks = new TestWebhooks(mock(), mock(), registrations, mock(), mock()); jest.useFakeTimers(); }); diff --git a/packages/cli/src/webhooks/test-webhooks.ts b/packages/cli/src/webhooks/test-webhooks.ts index 21511d4843..bf2fa6c9d8 100644 --- a/packages/cli/src/webhooks/test-webhooks.ts +++ b/packages/cli/src/webhooks/test-webhooks.ts @@ -16,6 +16,7 @@ import { WorkflowMissingIdError } from '@/errors/workflow-missing-id.error'; import type { IWorkflowDb } from '@/interfaces'; import { NodeTypes } from '@/node-types'; import { Push } from '@/push'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { OrchestrationService } from '@/services/orchestration.service'; import { removeTrailingSlash } from '@/utils'; import type { TestWebhookRegistration } from '@/webhooks/test-webhook-registrations.service'; @@ -41,6 +42,7 @@ export class TestWebhooks implements IWebhookManager { private readonly nodeTypes: NodeTypes, private readonly registrations: TestWebhookRegistrationsService, private readonly orchestrationService: OrchestrationService, + private readonly publisher: Publisher, ) {} private timeouts: { [webhookKey: string]: NodeJS.Timeout } = {}; @@ -156,8 +158,10 @@ export class TestWebhooks implements IWebhookManager { pushRef && !this.push.getBackend().hasPushRef(pushRef) ) { - const payload = { webhookKey: key, workflowEntity, pushRef }; - void this.orchestrationService.publish('clear-test-webhooks', payload); + void this.publisher.publishCommand({ + command: 'clear-test-webhooks', + payload: { webhookKey: key, workflowEntity, pushRef }, + }); return; } diff --git a/packages/cli/src/webhooks/waiting-forms.ts b/packages/cli/src/webhooks/waiting-forms.ts new file mode 100644 index 0000000000..5a491c1fb3 --- /dev/null +++ b/packages/cli/src/webhooks/waiting-forms.ts @@ -0,0 +1,145 @@ +import axios from 'axios'; +import type express from 'express'; +import { FORM_NODE_TYPE, sleep, Workflow } from 'n8n-workflow'; +import { Service } from 'typedi'; + +import { ConflictError } from '@/errors/response-errors/conflict.error'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import type { IExecutionResponse } from '@/interfaces'; +import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; + +import type { IWebhookResponseCallbackData, WaitingWebhookRequest } from './webhook.types'; + +@Service() +export class WaitingForms extends WaitingWebhooks { + protected override includeForms = true; + + protected override logReceivedWebhook(method: string, executionId: string) { + this.logger.debug(`Received waiting-form "${method}" for execution "${executionId}"`); + } + + protected disableNode(execution: IExecutionResponse, method?: string) { + if (method === 'POST') { + execution.data.executionData!.nodeExecutionStack[0].node.disabled = true; + } + } + + private getWorkflow(execution: IExecutionResponse) { + const { workflowData } = execution; + return new Workflow({ + id: workflowData.id, + name: workflowData.name, + nodes: workflowData.nodes, + connections: workflowData.connections, + active: workflowData.active, + nodeTypes: this.nodeTypes, + staticData: workflowData.staticData, + settings: workflowData.settings, + }); + } + + private async reloadForm(req: WaitingWebhookRequest, res: express.Response) { + try { + await sleep(1000); + + const url = `${req.protocol}://${req.get('host')}${req.originalUrl}`; + const page = await axios({ url }); + + if (page) { + res.send(` + + `); + } + } catch (error) {} + } + + async executeWebhook( + req: WaitingWebhookRequest, + res: express.Response, + ): Promise { + const { path: executionId, suffix } = req.params; + + this.logReceivedWebhook(req.method, executionId); + + // Reset request parameters + req.params = {} as WaitingWebhookRequest['params']; + + const execution = await this.getExecution(executionId); + + if (!execution) { + throw new NotFoundError(`The execution "${executionId}" does not exist.`); + } + + if (execution.data.resultData.error) { + const message = `The execution "${executionId}" has finished with error.`; + this.logger.debug(message, { error: execution.data.resultData.error }); + throw new ConflictError(message); + } + + if (execution.status === 'running') { + if (this.includeForms && req.method === 'GET') { + await this.reloadForm(req, res); + return { noWebhookResponse: true }; + } + + throw new ConflictError(`The execution "${executionId}" is running already.`); + } + + let completionPage; + if (execution.finished) { + const workflow = this.getWorkflow(execution); + + const parentNodes = workflow.getParentNodes( + execution.data.resultData.lastNodeExecuted as string, + ); + + const lastNodeExecuted = execution.data.resultData.lastNodeExecuted as string; + const lastNode = workflow.nodes[lastNodeExecuted]; + + if ( + !lastNode.disabled && + lastNode.type === FORM_NODE_TYPE && + lastNode.parameters.operation === 'completion' + ) { + completionPage = lastNodeExecuted; + } else { + completionPage = Object.keys(workflow.nodes).find((nodeName) => { + const node = workflow.nodes[nodeName]; + return ( + parentNodes.includes(nodeName) && + !node.disabled && + node.type === FORM_NODE_TYPE && + node.parameters.operation === 'completion' + ); + }); + } + + if (!completionPage) { + res.render('form-trigger-completion', { + title: 'Form Submitted', + message: 'Your response has been recorded', + formTitle: 'Form Submitted', + }); + + return { + noWebhookResponse: true, + }; + } + } + + const targetNode = completionPage || (execution.data.resultData.lastNodeExecuted as string); + + return await this.getWebhookExecutionData({ + execution, + req, + res, + lastNodeExecuted: targetNode, + executionId, + suffix, + }); + } +} diff --git a/packages/cli/src/webhooks/waiting-webhooks.ts b/packages/cli/src/webhooks/waiting-webhooks.ts index e644c065f3..9529d04c04 100644 --- a/packages/cli/src/webhooks/waiting-webhooks.ts +++ b/packages/cli/src/webhooks/waiting-webhooks.ts @@ -1,9 +1,11 @@ import type express from 'express'; import { + FORM_NODE_TYPE, type INodes, type IWorkflowBase, NodeHelpers, SEND_AND_WAIT_OPERATION, + WAIT_NODE_TYPE, Workflow, } from 'n8n-workflow'; import { Service } from 'typedi'; @@ -34,7 +36,7 @@ export class WaitingWebhooks implements IWebhookManager { constructor( protected readonly logger: Logger, - private readonly nodeTypes: NodeTypes, + protected readonly nodeTypes: NodeTypes, private readonly executionRepository: ExecutionRepository, ) {} @@ -58,7 +60,7 @@ export class WaitingWebhooks implements IWebhookManager { ); } - private getWorkflow(workflowData: IWorkflowBase) { + private createWorkflow(workflowData: IWorkflowBase) { return new Workflow({ id: workflowData.id, name: workflowData.name, @@ -71,6 +73,13 @@ export class WaitingWebhooks implements IWebhookManager { }); } + protected async getExecution(executionId: string) { + return await this.executionRepository.findSingleExecution(executionId, { + includeData: true, + unflattenData: true, + }); + } + async executeWebhook( req: WaitingWebhookRequest, res: express.Response, @@ -82,26 +91,25 @@ export class WaitingWebhooks implements IWebhookManager { // Reset request parameters req.params = {} as WaitingWebhookRequest['params']; - const execution = await this.executionRepository.findSingleExecution(executionId, { - includeData: true, - unflattenData: true, - }); + const execution = await this.getExecution(executionId); if (!execution) { - throw new NotFoundError(`The execution "${executionId} does not exist.`); + throw new NotFoundError(`The execution "${executionId}" does not exist.`); } if (execution.status === 'running') { - throw new ConflictError(`The execution "${executionId} is running already.`); + throw new ConflictError(`The execution "${executionId}" is running already.`); } if (execution.data?.resultData?.error) { - throw new ConflictError(`The execution "${executionId} has finished already.`); + const message = `The execution "${executionId}" has finished with error.`; + this.logger.debug(message, { error: execution.data.resultData.error }); + throw new ConflictError(message); } if (execution.finished) { const { workflowData } = execution; - const { nodes } = this.getWorkflow(workflowData); + const { nodes } = this.createWorkflow(workflowData); if (this.isSendAndWaitRequest(nodes, suffix)) { res.render('send-and-wait-no-action-required', { isTestWebhook: false }); return { noWebhookResponse: true }; @@ -112,6 +120,31 @@ export class WaitingWebhooks implements IWebhookManager { const lastNodeExecuted = execution.data.resultData.lastNodeExecuted as string; + return await this.getWebhookExecutionData({ + execution, + req, + res, + lastNodeExecuted, + executionId, + suffix, + }); + } + + protected async getWebhookExecutionData({ + execution, + req, + res, + lastNodeExecuted, + executionId, + suffix, + }: { + execution: IExecutionResponse; + req: WaitingWebhookRequest; + res: express.Response; + lastNodeExecuted: string; + executionId: string; + suffix?: string; + }): Promise { // Set the node as disabled so that the data does not get executed again as it would result // in starting the wait all over again this.disableNode(execution, req.method); @@ -123,7 +156,7 @@ export class WaitingWebhooks implements IWebhookManager { execution.data.resultData.runData[lastNodeExecuted].pop(); const { workflowData } = execution; - const workflow = this.getWorkflow(workflowData); + const workflow = this.createWorkflow(workflowData); const workflowStartNode = workflow.getNode(lastNodeExecuted); if (workflowStartNode === null) { @@ -146,13 +179,30 @@ export class WaitingWebhooks implements IWebhookManager { if (webhookData === undefined) { // If no data got found it means that the execution can not be started via a webhook. // Return 404 because we do not want to give any data if the execution exists or not. + const errorMessage = `The workflow for execution "${executionId}" does not contain a waiting webhook with a matching path/method.`; + if (this.isSendAndWaitRequest(workflow.nodes, suffix)) { res.render('send-and-wait-no-action-required', { isTestWebhook: false }); return { noWebhookResponse: true }; - } else { - const errorMessage = `The workflow for execution "${executionId}" does not contain a waiting webhook with a matching path/method.`; - throw new NotFoundError(errorMessage); } + + if (!execution.data.resultData.error && execution.status === 'waiting') { + const childNodes = workflow.getChildNodes( + execution.data.resultData.lastNodeExecuted as string, + ); + + const hasChildForms = childNodes.some( + (node) => + workflow.nodes[node].type === FORM_NODE_TYPE || + workflow.nodes[node].type === WAIT_NODE_TYPE, + ); + + if (hasChildForms) { + return { noWebhookResponse: true }; + } + } + + throw new NotFoundError(errorMessage); } const runExecutionData = execution.data; diff --git a/packages/cli/src/webhooks/webhook-helpers.ts b/packages/cli/src/webhooks/webhook-helpers.ts index 5ff770acfb..72628b8351 100644 --- a/packages/cli/src/webhooks/webhook-helpers.ts +++ b/packages/cli/src/webhooks/webhook-helpers.ts @@ -34,7 +34,11 @@ import { BINARY_ENCODING, createDeferredPromise, ErrorReporterProxy as ErrorReporter, + ErrorReporterProxy, + ExecutionCancelledError, + FORM_NODE_TYPE, NodeHelpers, + NodeOperationError, } from 'n8n-workflow'; import { finished } from 'stream/promises'; import { Container } from 'typedi'; @@ -120,7 +124,7 @@ export async function executeWebhook( ); if (nodeType === undefined) { const errorMessage = `The type of the webhook node "${workflowStartNode.name}" is not known`; - responseCallback(new Error(errorMessage), {}); + responseCallback(new ApplicationError(errorMessage), {}); throw new InternalServerError(errorMessage); } @@ -143,14 +147,37 @@ export async function executeWebhook( } // Get the responseMode - const responseMode = workflow.expression.getSimpleParameterValue( - workflowStartNode, - webhookData.webhookDescription.responseMode, - executionMode, - additionalKeys, - undefined, - 'onReceived', - ) as WebhookResponseMode; + let responseMode; + + // if this is n8n FormTrigger node, check if there is a Form node in child nodes, + // if so, set 'responseMode' to 'formPage' to redirect to URL of that Form later + if (nodeType.description.name === 'formTrigger') { + const connectedNodes = workflow.getChildNodes(workflowStartNode.name); + let hasNextPage = false; + for (const nodeName of connectedNodes) { + const node = workflow.nodes[nodeName]; + if (node.type === FORM_NODE_TYPE && !node.disabled) { + hasNextPage = true; + break; + } + } + + if (hasNextPage) { + responseMode = 'formPage'; + } + } + + if (!responseMode) { + responseMode = workflow.expression.getSimpleParameterValue( + workflowStartNode, + webhookData.webhookDescription.responseMode, + executionMode, + additionalKeys, + undefined, + 'onReceived', + ) as WebhookResponseMode; + } + const responseCode = workflow.expression.getSimpleParameterValue( workflowStartNode, webhookData.webhookDescription.responseCode as string, @@ -169,12 +196,12 @@ export async function executeWebhook( 'firstEntryJson', ); - if (!['onReceived', 'lastNode', 'responseNode'].includes(responseMode)) { + if (!['onReceived', 'lastNode', 'responseNode', 'formPage'].includes(responseMode)) { // If the mode is not known we error. Is probably best like that instead of using // the default that people know as early as possible (probably already testing phase) // that something does not resolve properly. const errorMessage = `The response mode '${responseMode}' is not valid!`; - responseCallback(new Error(errorMessage), {}); + responseCallback(new ApplicationError(errorMessage), {}); throw new InternalServerError(errorMessage); } @@ -242,8 +269,26 @@ export async function executeWebhook( }); } catch (err) { // Send error response to webhook caller - const errorMessage = 'Workflow Webhook Error: Workflow could not be started!'; - responseCallback(new Error(errorMessage), {}); + const webhookType = ['formTrigger', 'form'].includes(nodeType.description.name) + ? 'Form' + : 'Webhook'; + let errorMessage = `Workflow ${webhookType} Error: Workflow could not be started!`; + + // if workflow started manually, show an actual error message + if (err instanceof NodeOperationError && err.type === 'manual-form-test') { + errorMessage = err.message; + } + + ErrorReporterProxy.error(err, { + extra: { + nodeName: workflowStartNode.name, + nodeType: workflowStartNode.type, + nodeVersion: workflowStartNode.typeVersion, + workflowId: workflow.id, + }, + }); + + responseCallback(new ApplicationError(errorMessage), {}); didSendResponse = true; // Add error to execution data that it can be logged and send to Editor-UI @@ -487,6 +532,12 @@ export async function executeWebhook( responsePromise, ); + if (responseMode === 'formPage' && !didSendResponse) { + res.redirect(`${additionalData.formWaitingBaseUrl}/${executionId}`); + process.nextTick(() => res.end()); + didSendResponse = true; + } + Container.get(Logger).debug( `Started execution of workflow "${workflow.name}" from webhook with execution ID ${executionId}`, { executionId }, @@ -562,7 +613,7 @@ export async function executeWebhook( // Return the JSON data of the first entry if (returnData.data!.main[0]![0] === undefined) { - responseCallback(new Error('No item to return got found'), {}); + responseCallback(new ApplicationError('No item to return got found'), {}); didSendResponse = true; return undefined; } @@ -616,13 +667,13 @@ export async function executeWebhook( data = returnData.data!.main[0]![0]; if (data === undefined) { - responseCallback(new Error('No item was found to return'), {}); + responseCallback(new ApplicationError('No item was found to return'), {}); didSendResponse = true; return undefined; } if (data.binary === undefined) { - responseCallback(new Error('No binary data was found to return'), {}); + responseCallback(new ApplicationError('No binary data was found to return'), {}); didSendResponse = true; return undefined; } @@ -637,7 +688,10 @@ export async function executeWebhook( ); if (responseBinaryPropertyName === undefined && !didSendResponse) { - responseCallback(new Error("No 'responseBinaryPropertyName' is set"), {}); + responseCallback( + new ApplicationError("No 'responseBinaryPropertyName' is set"), + {}, + ); didSendResponse = true; } @@ -646,7 +700,7 @@ export async function executeWebhook( ]; if (binaryData === undefined && !didSendResponse) { responseCallback( - new Error( + new ApplicationError( `The binary property '${responseBinaryPropertyName}' which should be returned does not exist`, ), {}, @@ -703,7 +757,9 @@ export async function executeWebhook( ); } - throw new InternalServerError(e.message); + const internalServerError = new InternalServerError(e.message); + if (e instanceof ExecutionCancelledError) internalServerError.level = 'warning'; + throw internalServerError; }); } return executionId; diff --git a/packages/cli/src/webhooks/webhook-server.ts b/packages/cli/src/webhooks/webhook-server.ts index d54f39f2cf..263375325b 100644 --- a/packages/cli/src/webhooks/webhook-server.ts +++ b/packages/cli/src/webhooks/webhook-server.ts @@ -3,8 +3,4 @@ import { Service } from 'typedi'; import { AbstractServer } from '@/abstract-server'; @Service() -export class WebhookServer extends AbstractServer { - constructor() { - super('webhook'); - } -} +export class WebhookServer extends AbstractServer {} diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index 8d1e147e85..02e0b94afd 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-shadow */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { GlobalConfig } from '@n8n/config'; import { InstanceSettings, WorkflowExecute } from 'n8n-core'; import type { ExecutionError, @@ -29,7 +28,7 @@ import { ExternalHooks } from '@/external-hooks'; import { Logger } from '@/logging/logger.service'; import { NodeTypes } from '@/node-types'; import type { ScalingService } from '@/scaling/scaling.service'; -import type { Job, JobData, JobResult } from '@/scaling/scaling.types'; +import type { Job, JobData } from '@/scaling/scaling.types'; import { PermissionChecker } from '@/user-management/permission-checker'; import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import * as WorkflowHelpers from '@/workflow-helpers'; @@ -64,7 +63,7 @@ export class WorkflowRunner { executionId: string, hooks?: WorkflowHooks, ) { - ErrorReporter.error(error); + ErrorReporter.error(error, { executionId }); const isQueueMode = config.getEnv('executions.mode') === 'queue'; @@ -427,56 +426,9 @@ export class WorkflowRunner { reject(error); }); - const jobData: Promise = job.finished(); - - const { queueRecoveryInterval } = Container.get(GlobalConfig).queue.bull; - - const racingPromises: Array> = [jobData]; - - let clearWatchdogInterval; - if (queueRecoveryInterval > 0) { - /** *********************************************** - * Long explanation about what this solves: * - * This only happens in a very specific scenario * - * when Redis crashes and recovers shortly * - * but during this time, some execution(s) * - * finished. The end result is that the main * - * process will wait indefinitely and never * - * get a response. This adds an active polling to* - * the queue that allows us to identify that the * - * execution finished and get information from * - * the database. * - ************************************************ */ - let watchDogInterval: NodeJS.Timeout | undefined; - - const watchDog: Promise = new Promise((res) => { - watchDogInterval = setInterval(async () => { - const currentJob = await this.scalingService.getJob(job.id); - // When null means job is finished (not found in queue) - if (currentJob === null) { - // Mimic worker's success message - res({ success: true }); - } - }, queueRecoveryInterval * 1000); - }); - - racingPromises.push(watchDog); - - clearWatchdogInterval = () => { - if (watchDogInterval) { - clearInterval(watchDogInterval); - watchDogInterval = undefined; - } - }; - } - try { - await Promise.race(racingPromises); - if (clearWatchdogInterval !== undefined) { - clearWatchdogInterval(); - } + await job.finished(); } catch (error) { - ErrorReporter.error(error); // We use "getWorkflowHooksWorkerExecuter" as "getWorkflowHooksWorkerMain" does not contain the // "workflowExecuteAfter" which we require. const hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( @@ -486,9 +438,6 @@ export class WorkflowRunner { { retryOf: data.retryOf ? data.retryOf.toString() : undefined }, ); this.logger.error(`Problem with execution ${executionId}: ${error.message}. Aborting.`); - if (clearWatchdogInterval !== undefined) { - clearWatchdogInterval(); - } await this.processError(error, new Date(), data.executionMode, executionId, hooks); reject(error); diff --git a/packages/cli/templates/form-trigger-completion.handlebars b/packages/cli/templates/form-trigger-completion.handlebars new file mode 100644 index 0000000000..761d09937b --- /dev/null +++ b/packages/cli/templates/form-trigger-completion.handlebars @@ -0,0 +1,74 @@ + + + + + + + + + {{formTitle}} + + + + + +
+
+
+
+

{{title}}

+

{{message}}

+
+
+ {{#if appendAttribution}} +
+ + Form automated with + + + + + + + +
+ {{/if}} +
+
+ + + \ No newline at end of file diff --git a/packages/cli/templates/form-trigger.handlebars b/packages/cli/templates/form-trigger.handlebars index 5493a76e7f..08f6d2e290 100644 --- a/packages/cli/templates/form-trigger.handlebars +++ b/packages/cli/templates/form-trigger.handlebars @@ -315,7 +315,7 @@
{{#if testRun}}
-

This is test version of your form. Use it only for testing your Form Trigger.

+

This is test version of your form

{{/if}} @@ -428,7 +428,7 @@ d='M304 48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zm0 416a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM48 304a48 48 0 1 0 0-96 48 48 0 1 0 0 96zm464-48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM142.9 437A48 48 0 1 0 75 369.1 48 48 0 1 0 142.9 437zm0-294.2A48 48 0 1 0 75 75a48 48 0 1 0 67.9 67.9zM369.1 437A48 48 0 1 0 437 369.1 48 48 0 1 0 369.1 437z' /> - Submit form + {{ buttonLabel }} {{else}} @@ -719,6 +719,10 @@ } if (response.status === 200) { + if(response.redirected) { + window.location.replace(response.url); + return; + } const redirectUrl = document.getElementById("redirectUrl"); if (redirectUrl) { window.location.replace(redirectUrl.href); @@ -731,7 +735,7 @@ document.querySelector('#submitted-form').style.display = 'block'; document.querySelector('#submitted-header').textContent = 'Problem submitting response'; document.querySelector('#submitted-content').textContent = - 'An error occurred in the workflow handling this form'; + 'Please try again or contact support if the problem persists'; } return; @@ -747,6 +751,18 @@ .catch(function (error) { console.error('Error:', error); }); + + const isWaitingForm = window.location.href.includes('form-waiting'); + if(isWaitingForm) { + const interval = setInterval(function() { + const isSubmited = document.querySelector('#submitted-form').style.display; + if(isSubmited === 'block') { + clearInterval(interval); + return; + } + window.location.reload(); + }, 2000); + } } }); diff --git a/packages/cli/templates/saml-connection-test-failed.handlebars b/packages/cli/templates/saml-connection-test-failed.handlebars new file mode 100644 index 0000000000..f93cea5c2d --- /dev/null +++ b/packages/cli/templates/saml-connection-test-failed.handlebars @@ -0,0 +1,30 @@ + + + n8n - SAML Connection Test Result + + + +
+

SAML Connection Test failed

+

{{#if message}}{{message}}{{else}}A common issue could be that no email attribute is set{{/if}}

+ +

+ {{#with attributes}} +

Here are the attributes returned by your SAML IdP:

+
    +
  • Email: {{#if email}}{{email}}{{else}}(n/a){{/if}}
    • +
  • First Name: {{#if firstName}}{{firstName}}{{else}}(n/a){{/if}}
    • +
  • Last Name: {{#if lastName}}{{lastName}}{{else}}(n/a){{/if}}
    • +
  • UPN: {{#if userPrincipalName}}{{userPrincipalName}}{{else}}(n/a){{/if}}
    • + {{/with}} +
+
+ + diff --git a/packages/cli/templates/saml-connection-test-success.handlebars b/packages/cli/templates/saml-connection-test-success.handlebars new file mode 100644 index 0000000000..e65d29483d --- /dev/null +++ b/packages/cli/templates/saml-connection-test-success.handlebars @@ -0,0 +1,27 @@ + + + n8n - SAML Connection Test Result + + + +
+

SAML Connection Test was successful

+ +

+

Here are the attributes returned by your SAML IdP:

+
    +
  • Email: {{#if email}}{{email}}{{else}}(n/a){{/if}}
    • +
  • First Name: {{#if firstName}}{{firstName}}{{else}}(n/a){{/if}}
    • +
  • Last Name: {{#if lastName}}{{lastName}}{{else}}(n/a){{/if}}
    • +
  • UPN: {{#if userPrincipalName}}{{userPrincipalName}}{{else}}(n/a){{/if}}
    • +
+
+ + diff --git a/packages/cli/test/integration/active-workflow-manager.test.ts b/packages/cli/test/integration/active-workflow-manager.test.ts index d5d471ba60..8ea790ade7 100644 --- a/packages/cli/test/integration/active-workflow-manager.test.ts +++ b/packages/cli/test/integration/active-workflow-manager.test.ts @@ -1,4 +1,5 @@ import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; import { NodeApiError, NodeOperationError, Workflow } from 'n8n-workflow'; import type { IWebhookData, WorkflowActivateMode } from 'n8n-workflow'; import { Container } from 'typedi'; @@ -278,3 +279,72 @@ describe('addWebhooks()', () => { expect(webhookService.storeWebhook).toHaveBeenCalledTimes(1); }); }); + +describe('shouldAddWebhooks', () => { + describe('if leader', () => { + const activeWorkflowManager = new ActiveWorkflowManager( + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock({ isLeader: true, isFollower: false }), + mock(), + ); + + test('should return `true` for `init`', () => { + // ensure webhooks are populated on init: https://github.com/n8n-io/n8n/pull/8830 + const result = activeWorkflowManager.shouldAddWebhooks('init'); + expect(result).toBe(true); + }); + + test('should return `false` for `leadershipChange`', () => { + const result = activeWorkflowManager.shouldAddWebhooks('leadershipChange'); + expect(result).toBe(false); + }); + + test('should return `true` for `update` or `activate`', () => { + const modes = ['update', 'activate'] as WorkflowActivateMode[]; + for (const mode of modes) { + const result = activeWorkflowManager.shouldAddWebhooks(mode); + expect(result).toBe(true); + } + }); + }); + + describe('if follower', () => { + const activeWorkflowManager = new ActiveWorkflowManager( + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock({ isLeader: false, isFollower: true }), + mock(), + ); + + test('should return `false` for `update` or `activate`', () => { + const modes = ['update', 'activate'] as WorkflowActivateMode[]; + for (const mode of modes) { + const result = activeWorkflowManager.shouldAddWebhooks(mode); + expect(result).toBe(false); + } + }); + }); +}); diff --git a/packages/cli/test/integration/collaboration/collaboration.service.test.ts b/packages/cli/test/integration/collaboration/collaboration.service.test.ts index a90424de87..df5f901f28 100644 --- a/packages/cli/test/integration/collaboration/collaboration.service.test.ts +++ b/packages/cli/test/integration/collaboration/collaboration.service.test.ts @@ -16,7 +16,7 @@ import { createWorkflow, shareWorkflowWithUsers } from '@test-integration/db/wor import * as testDb from '@test-integration/test-db'; describe('CollaborationService', () => { - mockInstance(Push, new Push(mock())); + mockInstance(Push, new Push(mock(), mock())); let pushService: Push; let collaborationService: CollaborationService; let owner: User; diff --git a/packages/cli/test/integration/commands/worker.cmd.test.ts b/packages/cli/test/integration/commands/worker.cmd.test.ts index 2326ed595a..ce3280aa48 100644 --- a/packages/cli/test/integration/commands/worker.cmd.test.ts +++ b/packages/cli/test/integration/commands/worker.cmd.test.ts @@ -1,6 +1,8 @@ process.argv[2] = 'worker'; +import { TaskRunnersConfig } from '@n8n/config'; import { BinaryDataService } from 'n8n-core'; +import Container from 'typedi'; import { Worker } from '@/commands/worker'; import config from '@/config'; @@ -11,10 +13,12 @@ import { ExternalSecretsManager } from '@/external-secrets/external-secrets-mana import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { Push } from '@/push'; +import { TaskRunnerProcess } from '@/runners/task-runner-process'; +import { TaskRunnerServer } from '@/runners/task-runner-server'; import { Publisher } from '@/scaling/pubsub/publisher.service'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import { ScalingService } from '@/scaling/scaling.service'; -import { OrchestrationWorkerService } from '@/services/orchestration/worker/orchestration.worker.service'; +import { OrchestrationService } from '@/services/orchestration.service'; import { Telemetry } from '@/telemetry'; import { setupTestCommand } from '@test-integration/utils/test-command'; @@ -22,6 +26,7 @@ import { mockInstance } from '../../shared/mocking'; config.set('executions.mode', 'queue'); config.set('binaryDataManager.availableModes', 'filesystem'); +Container.get(TaskRunnersConfig).disabled = false; mockInstance(LoadNodesAndCredentials); const binaryDataService = mockInstance(BinaryDataService); const externalHooks = mockInstance(ExternalHooks); @@ -30,7 +35,9 @@ const license = mockInstance(License, { loadCertStr: async () => '' }); const messageEventBus = mockInstance(MessageEventBus); const logStreamingEventRelay = mockInstance(LogStreamingEventRelay); const scalingService = mockInstance(ScalingService); -const orchestrationWorkerService = mockInstance(OrchestrationWorkerService); +const orchestrationService = mockInstance(OrchestrationService); +const taskRunnerServer = mockInstance(TaskRunnerServer); +const taskRunnerProcess = mockInstance(TaskRunnerProcess); mockInstance(Publisher); mockInstance(Subscriber); mockInstance(Telemetry); @@ -39,10 +46,10 @@ mockInstance(Push); const command = setupTestCommand(Worker); test('worker initializes all its components', async () => { - const worker = await command.run(); - expect(worker.queueModeId).toBeDefined(); - expect(worker.queueModeId).toContain('worker'); - expect(worker.queueModeId.length).toBeGreaterThan(15); + config.set('executions.mode', 'regular'); // should be overridden + + await command.run(); + expect(license.init).toHaveBeenCalledTimes(1); expect(binaryDataService.init).toHaveBeenCalledTimes(1); expect(externalHooks.init).toHaveBeenCalledTimes(1); @@ -51,6 +58,10 @@ test('worker initializes all its components', async () => { expect(scalingService.setupQueue).toHaveBeenCalledTimes(1); expect(scalingService.setupWorker).toHaveBeenCalledTimes(1); expect(logStreamingEventRelay.init).toHaveBeenCalledTimes(1); - expect(orchestrationWorkerService.init).toHaveBeenCalledTimes(1); + expect(orchestrationService.init).toHaveBeenCalledTimes(1); expect(messageEventBus.send).toHaveBeenCalledTimes(1); + expect(taskRunnerServer.start).toHaveBeenCalledTimes(1); + expect(taskRunnerProcess.start).toHaveBeenCalledTimes(1); + + expect(config.getEnv('executions.mode')).toBe('queue'); }); diff --git a/packages/cli/test/integration/controllers/invitation/invitation.controller.integration.test.ts b/packages/cli/test/integration/controllers/invitation/invitation.controller.integration.test.ts index 48663e1ef3..f1e8e2a1b9 100644 --- a/packages/cli/test/integration/controllers/invitation/invitation.controller.integration.test.ts +++ b/packages/cli/test/integration/controllers/invitation/invitation.controller.integration.test.ts @@ -246,6 +246,7 @@ describe('InvitationController', () => { const { user } = response.body.data[0]; expect(user.inviteAcceptUrl).toBeDefined(); + expect(user).toHaveProperty('role', 'global:member'); const inviteUrl = new URL(user.inviteAcceptUrl); diff --git a/packages/cli/test/integration/database/repositories/execution.repository.test.ts b/packages/cli/test/integration/database/repositories/execution.repository.test.ts index 52884bd3e6..1b50415686 100644 --- a/packages/cli/test/integration/database/repositories/execution.repository.test.ts +++ b/packages/cli/test/integration/database/repositories/execution.repository.test.ts @@ -1,3 +1,4 @@ +import { GlobalConfig } from '@n8n/config'; import Container from 'typedi'; import { ExecutionDataRepository } from '@/databases/repositories/execution-data.repository'; @@ -54,5 +55,38 @@ describe('ExecutionRepository', () => { }); expect(executionData?.data).toEqual('[{"resultData":"1"},{}]'); }); + + it('should not create execution if execution data insert fails', async () => { + const { type: dbType, sqlite: sqliteConfig } = Container.get(GlobalConfig).database; + // Do not run this test for the legacy sqlite driver + if (dbType === 'sqlite' && sqliteConfig.poolSize === 0) return; + + const executionRepo = Container.get(ExecutionRepository); + const executionDataRepo = Container.get(ExecutionDataRepository); + + const workflow = await createWorkflow({ settings: { executionOrder: 'v1' } }); + jest + .spyOn(executionDataRepo, 'createExecutionDataForExecution') + .mockRejectedValueOnce(new Error()); + + await expect( + async () => + await executionRepo.createNewExecution({ + workflowId: workflow.id, + data: { + //@ts-expect-error This is not needed for tests + resultData: {}, + }, + workflowData: workflow, + mode: 'manual', + startedAt: new Date(), + status: 'new', + finished: false, + }), + ).rejects.toThrow(); + + const executionEntities = await executionRepo.find(); + expect(executionEntities).toBeEmptyArray(); + }); }); }); diff --git a/packages/cli/test/integration/debug.controller.test.ts b/packages/cli/test/integration/debug.controller.test.ts index 723edea58a..8ab58bd1a0 100644 --- a/packages/cli/test/integration/debug.controller.test.ts +++ b/packages/cli/test/integration/debug.controller.test.ts @@ -1,9 +1,11 @@ +import { InstanceSettings } from 'n8n-core'; +import Container from 'typedi'; + import { ActiveWorkflowManager } from '@/active-workflow-manager'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { generateNanoId } from '@/databases/utils/generators'; -import { MultiMainSetup } from '@/services/orchestration/main/multi-main-setup.ee'; -import { OrchestrationService } from '@/services/orchestration.service'; +import { MultiMainSetup } from '@/scaling/multi-main-setup.ee'; import { createOwner } from './shared/db/users'; import { randomName } from './shared/random'; @@ -14,6 +16,8 @@ import { mockInstance } from '../shared/mocking'; describe('DebugController', () => { const workflowRepository = mockInstance(WorkflowRepository); const activeWorkflowManager = mockInstance(ActiveWorkflowManager); + const instanceSettings = Container.get(InstanceSettings); + instanceSettings.markAsLeader(); let testServer = setupTestServer({ endpointGroups: ['debug'] }); let ownerAgent: SuperAgentTest; @@ -30,7 +34,7 @@ describe('DebugController', () => { const webhooks = [{ id: workflowId, name: randomName() }] as WorkflowEntity[]; const triggersAndPollers = [{ id: workflowId, name: randomName() }] as WorkflowEntity[]; const activationErrors = { [workflowId]: 'Failed to activate' }; - const instanceId = 'main-71JdWtq306epIFki'; + const { instanceId } = instanceSettings; const leaderKey = 'some-leader-key'; workflowRepository.findIn.mockResolvedValue(triggersAndPollers); @@ -38,9 +42,7 @@ describe('DebugController', () => { activeWorkflowManager.allActiveInMemory.mockReturnValue([workflowId]); activeWorkflowManager.getAllWorkflowActivationErrors.mockResolvedValue(activationErrors); - jest.spyOn(OrchestrationService.prototype, 'instanceId', 'get').mockReturnValue(instanceId); jest.spyOn(MultiMainSetup.prototype, 'fetchLeaderKey').mockResolvedValue(leaderKey); - jest.spyOn(OrchestrationService.prototype, 'isLeader', 'get').mockReturnValue(true); const response = await ownerAgent.get('/debug/multi-main-setup').expect(200); diff --git a/packages/cli/test/integration/eventbus.ee.test.ts b/packages/cli/test/integration/eventbus.ee.test.ts index 9b12cc53d5..c2b6a7f23c 100644 --- a/packages/cli/test/integration/eventbus.ee.test.ts +++ b/packages/cli/test/integration/eventbus.ee.test.ts @@ -22,6 +22,7 @@ import type { MessageEventBusDestinationSentry } from '@/eventbus/message-event- import type { MessageEventBusDestinationSyslog } from '@/eventbus/message-event-bus-destination/message-event-bus-destination-syslog.ee'; import type { MessageEventBusDestinationWebhook } from '@/eventbus/message-event-bus-destination/message-event-bus-destination-webhook.ee'; import { ExecutionRecoveryService } from '@/executions/execution-recovery.service'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; import { createUser } from './shared/db/users'; import type { SuperAgentTest } from './shared/types'; @@ -34,6 +35,8 @@ const mockedAxios = axios as jest.Mocked; jest.mock('syslog-client'); const mockedSyslog = syslog as jest.Mocked; +mockInstance(Publisher); + let owner: User; let authOwnerAgent: SuperAgentTest; diff --git a/packages/cli/test/integration/execution.service.integration.test.ts b/packages/cli/test/integration/execution.service.integration.test.ts index 22d0d65754..4d7144cd4d 100644 --- a/packages/cli/test/integration/execution.service.integration.test.ts +++ b/packages/cli/test/integration/execution.service.integration.test.ts @@ -563,10 +563,10 @@ describe('ExecutionService', () => { { ...summaryShape, annotation: { - tags: [ + tags: expect.arrayContaining([ expect.objectContaining({ name: 'tag1' }), expect.objectContaining({ name: 'tag2' }), - ], + ]), vote: 'up', }, }, @@ -646,10 +646,10 @@ describe('ExecutionService', () => { { ...summaryShape, annotation: { - tags: [ + tags: expect.arrayContaining([ expect.objectContaining({ name: 'tag1' }), expect.objectContaining({ name: 'tag2' }), - ], + ]), vote: 'up', }, }, @@ -691,10 +691,10 @@ describe('ExecutionService', () => { { ...summaryShape, annotation: { - tags: [ + tags: expect.arrayContaining([ expect.objectContaining({ name: 'tag1' }), expect.objectContaining({ name: 'tag2' }), - ], + ]), vote: 'up', }, }, diff --git a/packages/cli/test/integration/external-secrets/external-secrets.api.test.ts b/packages/cli/test/integration/external-secrets/external-secrets.api.test.ts index b3560b9262..c36340108e 100644 --- a/packages/cli/test/integration/external-secrets/external-secrets.api.test.ts +++ b/packages/cli/test/integration/external-secrets/external-secrets.api.test.ts @@ -18,7 +18,7 @@ import { MockProviders, TestFailProvider, } from '../../shared/external-secrets/utils'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance, mockLogger } from '../../shared/mocking'; import { createOwner, createUser } from '../shared/db/users'; import type { SuperAgentTest } from '../shared/types'; import { setupTestServer } from '../shared/utils'; @@ -52,17 +52,20 @@ async function getExternalSecretsSettings(): Promise(); +const logger = mockLogger(); + const resetManager = async () => { Container.get(ExternalSecretsManager).shutdown(); Container.set( ExternalSecretsManager, new ExternalSecretsManager( - mock(), + logger, Container.get(SettingsRepository), Container.get(License), mockProvidersInstance, Container.get(Cipher), eventService, + mock(), ), ); @@ -107,6 +110,18 @@ beforeAll(async () => { const member = await createUser(); authMemberAgent = testServer.authAgentFor(member); config.set('userManagement.isInstanceOwnerSetUp', true); + Container.set( + ExternalSecretsManager, + new ExternalSecretsManager( + logger, + Container.get(SettingsRepository), + Container.get(License), + mockProvidersInstance, + Container.get(Cipher), + eventService, + mock(), + ), + ); }); beforeEach(async () => { diff --git a/packages/cli/test/integration/license.api.test.ts b/packages/cli/test/integration/license.api.test.ts index ff40a699ea..a06408c6cb 100644 --- a/packages/cli/test/integration/license.api.test.ts +++ b/packages/cli/test/integration/license.api.test.ts @@ -116,7 +116,7 @@ describe('POST /license/renew', () => { const DEFAULT_LICENSE_RESPONSE: { data: ILicenseReadResponse } = { data: { usage: { - executions: { + activeWorkflowTriggers: { value: 0, limit: -1, warningThreshold: 0.8, @@ -132,7 +132,7 @@ const DEFAULT_LICENSE_RESPONSE: { data: ILicenseReadResponse } = { const DEFAULT_POST_RESPONSE: { data: ILicensePostResponse } = { data: { usage: { - executions: { + activeWorkflowTriggers: { value: 0, limit: -1, warningThreshold: 0.8, diff --git a/packages/cli/test/integration/mfa/mfa.api.test.ts b/packages/cli/test/integration/mfa/mfa.api.test.ts index 0062f87e89..3f19632506 100644 --- a/packages/cli/test/integration/mfa/mfa.api.test.ts +++ b/packages/cli/test/integration/mfa/mfa.api.test.ts @@ -5,9 +5,12 @@ import { AuthService } from '@/auth/auth.service'; import config from '@/config'; import type { User } from '@/databases/entities/user'; import { AuthUserRepository } from '@/databases/repositories/auth-user.repository'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import { ExternalHooks } from '@/external-hooks'; import { TOTPService } from '@/mfa/totp.service'; +import { mockInstance } from '@test/mocking'; -import { createUser, createUserWithMfaEnabled } from '../shared/db/users'; +import { createOwner, createUser, createUserWithMfaEnabled } from '../shared/db/users'; import { randomValidPassword, uniqueId } from '../shared/random'; import * as testDb from '../shared/test-db'; import * as utils from '../shared/utils'; @@ -16,6 +19,8 @@ jest.mock('@/telemetry'); let owner: User; +const externalHooks = mockInstance(ExternalHooks); + const testServer = utils.setupTestServer({ endpointGroups: ['mfa', 'auth', 'me', 'passwordReset'], }); @@ -23,7 +28,9 @@ const testServer = utils.setupTestServer({ beforeEach(async () => { await testDb.truncate(['User']); - owner = await createUser({ role: 'global:owner' }); + owner = await createOwner(); + + externalHooks.run.mockReset(); config.set('userManagement.disabled', false); }); @@ -131,6 +138,27 @@ describe('Enable MFA setup', () => { expect(user.mfaRecoveryCodes).toBeDefined(); expect(user.mfaSecret).toBeDefined(); }); + + test('POST /enable should not enable MFA if pre check fails', async () => { + // This test is to make sure owners verify their email before enabling MFA in cloud + + const response = await testServer.authAgentFor(owner).get('/mfa/qr').expect(200); + + const { secret } = response.body.data; + const token = new TOTPService().generateTOTP(secret); + + await testServer.authAgentFor(owner).post('/mfa/verify').send({ token }).expect(200); + + externalHooks.run.mockRejectedValue(new BadRequestError('Error message')); + + await testServer.authAgentFor(owner).post('/mfa/enable').send({ token }).expect(400); + + const user = await Container.get(AuthUserRepository).findOneOrFail({ + where: {}, + }); + + expect(user.mfaEnabled).toBe(false); + }); }); }); @@ -232,6 +260,28 @@ describe('Change password with MFA enabled', () => { }); }); +describe('MFA before enable checks', () => { + test('POST /can-enable should throw error if mfa.beforeSetup returns error', async () => { + externalHooks.run.mockRejectedValue(new BadRequestError('Error message')); + + await testServer.authAgentFor(owner).post('/mfa/can-enable').expect(400); + + expect(externalHooks.run).toHaveBeenCalledWith('mfa.beforeSetup', [ + expect.objectContaining(owner), + ]); + }); + + test('POST /can-enable should not throw error if mfa.beforeSetup does not exist', async () => { + externalHooks.run.mockResolvedValue(undefined); + + await testServer.authAgentFor(owner).post('/mfa/can-enable').expect(200); + + expect(externalHooks.run).toHaveBeenCalledWith('mfa.beforeSetup', [ + expect.objectContaining(owner), + ]); + }); +}); + describe('Login', () => { test('POST /login with email/password should succeed when mfa is disabled', async () => { const password = randomString(8); diff --git a/packages/cli/test/integration/pruning.service.test.ts b/packages/cli/test/integration/pruning.service.test.ts index c4d1957de0..5a53d70315 100644 --- a/packages/cli/test/integration/pruning.service.test.ts +++ b/packages/cli/test/integration/pruning.service.test.ts @@ -22,7 +22,7 @@ import { mockInstance } from '../shared/mocking'; describe('softDeleteOnPruningCycle()', () => { let pruningService: PruningService; - const instanceSettings = new InstanceSettings(); + const instanceSettings = new InstanceSettings(mock()); instanceSettings.markAsLeader(); const now = new Date(); @@ -38,6 +38,7 @@ describe('softDeleteOnPruningCycle()', () => { Container.get(ExecutionRepository), mockInstance(BinaryDataService), mock(), + mock(), ); workflow = await createWorkflow(); diff --git a/packages/cli/test/integration/public-api/credentials.test.ts b/packages/cli/test/integration/public-api/credentials.test.ts index 5574d4f3bf..953a6e1c95 100644 --- a/packages/cli/test/integration/public-api/credentials.test.ts +++ b/packages/cli/test/integration/public-api/credentials.test.ts @@ -6,7 +6,11 @@ import { CredentialsRepository } from '@/databases/repositories/credentials.repo import { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository'; import { createTeamProject } from '@test-integration/db/projects'; -import { affixRoleToSaveCredential, createCredentials } from '../shared/db/credentials'; +import { + affixRoleToSaveCredential, + createCredentials, + getCredentialSharings, +} from '../shared/db/credentials'; import { createMemberWithApiKey, createOwnerWithApiKey } from '../shared/db/users'; import { randomName } from '../shared/random'; import * as testDb from '../shared/test-db'; @@ -282,6 +286,41 @@ describe('PUT /credentials/:id/transfer', () => { expect(response.statusCode).toBe(204); }); + test('should transfer the right credential, not the first one it finds', async () => { + // ARRANGE + const [firstProject, secondProject] = await Promise.all([ + createTeamProject('first-project', owner), + createTeamProject('second-project', owner), + ]); + + const [firstCredential, secondCredential] = await Promise.all([ + createCredentials({ name: 'Test', type: 'test', data: '' }, firstProject), + createCredentials({ name: 'Test', type: 'test', data: '' }, firstProject), + ]); + + // ACT + const response = await authOwnerAgent.put(`/credentials/${secondCredential.id}/transfer`).send({ + destinationProjectId: secondProject.id, + }); + + // ASSERT + expect(response.statusCode).toBe(204); + + { + // second credential was moved + const sharings = await getCredentialSharings(secondCredential); + expect(sharings).toHaveLength(1); + expect(sharings[0]).toMatchObject({ projectId: secondProject.id }); + } + + { + // first credential was untouched + const sharings = await getCredentialSharings(firstCredential); + expect(sharings).toHaveLength(1); + expect(sharings[0]).toMatchObject({ projectId: firstProject.id }); + } + }); + test('if no destination project, should reject', async () => { /** * Arrange diff --git a/packages/cli/test/integration/runners/task-runner-process.test.ts b/packages/cli/test/integration/runners/task-runner-process.test.ts index f517ee6398..c893add440 100644 --- a/packages/cli/test/integration/runners/task-runner-process.test.ts +++ b/packages/cli/test/integration/runners/task-runner-process.test.ts @@ -1,4 +1,4 @@ -import { GlobalConfig } from '@n8n/config'; +import { TaskRunnersConfig } from '@n8n/config'; import Container from 'typedi'; import { TaskRunnerService } from '@/runners/runner-ws-server'; @@ -9,19 +9,26 @@ import { retryUntil } from '@test-integration/retry-until'; describe('TaskRunnerProcess', () => { const authToken = 'token'; - const globalConfig = Container.get(GlobalConfig); - globalConfig.taskRunners.authToken = authToken; - globalConfig.taskRunners.port = 0; // Use any port + const runnerConfig = Container.get(TaskRunnersConfig); + runnerConfig.disabled = false; + runnerConfig.mode = 'internal_childprocess'; + runnerConfig.authToken = authToken; + runnerConfig.port = 0; // Use any port const taskRunnerServer = Container.get(TaskRunnerServer); const runnerProcess = Container.get(TaskRunnerProcess); const taskBroker = Container.get(TaskBroker); const taskRunnerService = Container.get(TaskRunnerService); + const startLauncherSpy = jest.spyOn(runnerProcess, 'startLauncher'); + const startNodeSpy = jest.spyOn(runnerProcess, 'startNode'); + const killLauncherSpy = jest.spyOn(runnerProcess, 'killLauncher'); + const killNodeSpy = jest.spyOn(runnerProcess, 'killNode'); + beforeAll(async () => { await taskRunnerServer.start(); // Set the port to the actually used port - globalConfig.taskRunners.port = taskRunnerServer.port; + runnerConfig.port = taskRunnerServer.port; }); afterAll(async () => { @@ -30,6 +37,11 @@ describe('TaskRunnerProcess', () => { afterEach(async () => { await runnerProcess.stop(); + + startLauncherSpy.mockClear(); + startNodeSpy.mockClear(); + killLauncherSpy.mockClear(); + killNodeSpy.mockClear(); }); const getNumConnectedRunners = () => taskRunnerService.runnerConnections.size; @@ -78,14 +90,56 @@ describe('TaskRunnerProcess', () => { // @ts-expect-error private property runnerProcess.process?.kill('SIGKILL'); - // Assert - // Wait until the runner is running again - await retryUntil(() => expect(runnerProcess.isRunning).toBeTruthy()); - expect(runnerProcess.pid).not.toBe(processId); + // Wait until the runner has exited + await runnerProcess.runPromise; + // Assert // Wait until the runner has connected again await retryUntil(() => expect(getNumConnectedRunners()).toBe(1)); expect(getNumConnectedRunners()).toBe(1); expect(getNumRegisteredRunners()).toBe(1); + expect(runnerProcess.pid).not.toBe(processId); + }); + + it('should launch runner directly if not using a launcher', async () => { + runnerConfig.mode = 'internal_childprocess'; + + await runnerProcess.start(); + + expect(startLauncherSpy).toBeCalledTimes(0); + expect(startNodeSpy).toBeCalledTimes(1); + }); + + it('should use a launcher if configured', async () => { + runnerConfig.mode = 'internal_launcher'; + runnerConfig.launcherPath = 'node'; + + await runnerProcess.start(); + + expect(startLauncherSpy).toBeCalledTimes(1); + expect(startNodeSpy).toBeCalledTimes(0); + runnerConfig.mode = 'internal_childprocess'; + }); + + it('should kill the process directly if not using a launcher', async () => { + runnerConfig.mode = 'internal_childprocess'; + + await runnerProcess.start(); + await runnerProcess.stop(); + + expect(killLauncherSpy).toBeCalledTimes(0); + expect(killNodeSpy).toBeCalledTimes(1); + }); + + it('should kill the process using a launcher if configured', async () => { + runnerConfig.mode = 'internal_launcher'; + runnerConfig.launcherPath = 'node'; + + await runnerProcess.start(); + await runnerProcess.stop(); + + expect(killLauncherSpy).toBeCalledTimes(1); + expect(killNodeSpy).toBeCalledTimes(0); + runnerConfig.mode = 'internal_childprocess'; }); }); diff --git a/packages/cli/test/integration/security-audit/credentials-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/credentials-risk-reporter.test.ts index 4513beb6bb..b5b4c122df 100644 --- a/packages/cli/test/integration/security-audit/credentials-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/credentials-risk-reporter.test.ts @@ -1,7 +1,8 @@ +import type { SecurityConfig } from '@n8n/config'; +import { mock } from 'jest-mock-extended'; import Container from 'typedi'; import { v4 as uuid } from 'uuid'; -import config from '@/config'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import { ExecutionDataRepository } from '@/databases/repositories/execution-data.repository'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; @@ -15,10 +16,15 @@ import * as testDb from '../shared/test-db'; let securityAuditService: SecurityAuditService; +const securityConfig = mock({ daysAbandonedWorkflow: 90 }); + beforeAll(async () => { await testDb.init(); - securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository)); + securityAuditService = new SecurityAuditService( + Container.get(WorkflowRepository), + securityConfig, + ); }); beforeEach(async () => { @@ -154,7 +160,7 @@ test('should report credential in not recently executed workflow', async () => { const workflow = await Container.get(WorkflowRepository).save(workflowDetails); const date = new Date(); - date.setDate(date.getDate() - config.getEnv('security.audit.daysAbandonedWorkflow') - 1); + date.setDate(date.getDate() - securityConfig.daysAbandonedWorkflow - 1); const savedExecution = await Container.get(ExecutionRepository).save({ finished: true, @@ -223,7 +229,7 @@ test('should not report credentials in recently executed workflow', async () => const workflow = await Container.get(WorkflowRepository).save(workflowDetails); const date = new Date(); - date.setDate(date.getDate() - config.getEnv('security.audit.daysAbandonedWorkflow') + 1); + date.setDate(date.getDate() - securityConfig.daysAbandonedWorkflow + 1); const savedExecution = await Container.get(ExecutionRepository).save({ finished: true, diff --git a/packages/cli/test/integration/security-audit/database-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/database-risk-reporter.test.ts index d519f97a23..3aef57396b 100644 --- a/packages/cli/test/integration/security-audit/database-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/database-risk-reporter.test.ts @@ -1,3 +1,4 @@ +import { mock } from 'jest-mock-extended'; import Container from 'typedi'; import { v4 as uuid } from 'uuid'; @@ -18,7 +19,7 @@ let securityAuditService: SecurityAuditService; beforeAll(async () => { await testDb.init(); - securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository)); + securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository), mock()); }); beforeEach(async () => { diff --git a/packages/cli/test/integration/security-audit/filesystem-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/filesystem-risk-reporter.test.ts index 34bcb83b49..ceb306935f 100644 --- a/packages/cli/test/integration/security-audit/filesystem-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/filesystem-risk-reporter.test.ts @@ -1,3 +1,4 @@ +import { mock } from 'jest-mock-extended'; import Container from 'typedi'; import { v4 as uuid } from 'uuid'; @@ -13,7 +14,7 @@ let securityAuditService: SecurityAuditService; beforeAll(async () => { await testDb.init(); - securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository)); + securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository), mock()); }); beforeEach(async () => { diff --git a/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts index 4f355cbcbc..928667b518 100644 --- a/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts @@ -1,3 +1,4 @@ +import { mock } from 'jest-mock-extended'; import { NodeConnectionType } from 'n8n-workflow'; import Container from 'typedi'; import { v4 as uuid } from 'uuid'; @@ -23,7 +24,7 @@ let securityAuditService: SecurityAuditService; beforeAll(async () => { await testDb.init(); - securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository)); + securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository), mock()); simulateUpToDateInstance(); }); diff --git a/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts index 133a574d40..c1fb198b69 100644 --- a/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts @@ -1,3 +1,4 @@ +import { mock } from 'jest-mock-extended'; import { Container } from 'typedi'; import { v4 as uuid } from 'uuid'; @@ -24,7 +25,7 @@ let securityAuditService: SecurityAuditService; beforeAll(async () => { await testDb.init(); - securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository)); + securityAuditService = new SecurityAuditService(Container.get(WorkflowRepository), mock()); }); beforeEach(async () => { diff --git a/packages/cli/test/integration/shared/db/users.ts b/packages/cli/test/integration/shared/db/users.ts index 62f9f39a05..64c4d8ad85 100644 --- a/packages/cli/test/integration/shared/db/users.ts +++ b/packages/cli/test/integration/shared/db/users.ts @@ -1,17 +1,16 @@ import { hash } from 'bcryptjs'; -import { randomString } from 'n8n-workflow'; import Container from 'typedi'; import { AuthIdentity } from '@/databases/entities/auth-identity'; import { type GlobalRole, type User } from '@/databases/entities/user'; -import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; import { AuthIdentityRepository } from '@/databases/repositories/auth-identity.repository'; import { AuthUserRepository } from '@/databases/repositories/auth-user.repository'; import { UserRepository } from '@/databases/repositories/user.repository'; import { MfaService } from '@/mfa/mfa.service'; import { TOTPService } from '@/mfa/totp.service'; +import { PublicApiKeyService } from '@/services/public-api-key.service'; -import { randomApiKey, randomEmail, randomName, randomValidPassword } from '../random'; +import { randomEmail, randomName, randomValidPassword } from '../random'; // pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)` const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK'; @@ -81,17 +80,8 @@ export async function createUserWithMfaEnabled( }; } -const createApiKeyEntity = (user: User) => { - const apiKey = randomApiKey(); - return Container.get(ApiKeyRepository).create({ - userId: user.id, - label: randomString(10), - apiKey, - }); -}; - export const addApiKey = async (user: User) => { - return await Container.get(ApiKeyRepository).save(createApiKeyEntity(user)); + return await Container.get(PublicApiKeyService).createPublicApiKeyForUser(user); }; export async function createOwnerWithApiKey() { diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts index 4d4a207f94..78de2c1b25 100644 --- a/packages/cli/test/integration/shared/utils/index.ts +++ b/packages/cli/test/integration/shared/utils/index.ts @@ -32,7 +32,6 @@ export { setupTestServer } from './test-server'; export async function initActiveWorkflowManager() { mockInstance(OrchestrationService, { isMultiMainSetupEnabled: false, - shouldAddWebhooks: jest.fn().mockReturnValue(true), }); mockInstance(Push); diff --git a/packages/cli/test/integration/webhooks.test.ts b/packages/cli/test/integration/webhooks.test.ts index 165822aa84..7d7b5105cb 100644 --- a/packages/cli/test/integration/webhooks.test.ts +++ b/packages/cli/test/integration/webhooks.test.ts @@ -5,9 +5,9 @@ import type SuperAgentTest from 'supertest/lib/agent'; import Container from 'typedi'; import { ExternalHooks } from '@/external-hooks'; -import { WaitingForms } from '@/waiting-forms'; import { LiveWebhooks } from '@/webhooks/live-webhooks'; import { TestWebhooks } from '@/webhooks/test-webhooks'; +import { WaitingForms } from '@/webhooks/waiting-forms'; import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; import { WebhookServer } from '@/webhooks/webhook-server'; import type { IWebhookResponseCallbackData } from '@/webhooks/webhook.types'; diff --git a/packages/cli/test/setup-test-folder.ts b/packages/cli/test/setup-test-folder.ts index 94e45e6c90..997a0ec80f 100644 --- a/packages/cli/test/setup-test-folder.ts +++ b/packages/cli/test/setup-test-folder.ts @@ -14,5 +14,8 @@ process.env.N8N_USER_FOLDER = testDir; writeFileSync( join(testDir, '.n8n/config'), JSON.stringify({ encryptionKey: 'test_key', instanceId: '123' }), - 'utf-8', + { + encoding: 'utf-8', + mode: 0o600, + }, ); diff --git a/packages/cli/test/shared/mocking.ts b/packages/cli/test/shared/mocking.ts index 099988a896..129acb585c 100644 --- a/packages/cli/test/shared/mocking.ts +++ b/packages/cli/test/shared/mocking.ts @@ -25,5 +25,4 @@ export const mockEntityManager = (entityClass: Class) => { return entityManager; }; -export const mockLogger = () => - mock({ withScope: jest.fn().mockReturnValue(mock()) }); +export const mockLogger = () => mock({ scoped: jest.fn().mockReturnValue(mock()) }); diff --git a/packages/core/package.json b/packages/core/package.json index 95cf23efa6..0e010052b2 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "n8n-core", - "version": "1.63.0", + "version": "1.65.0", "description": "Core functionality of n8n", "main": "dist/index", "types": "dist/index.d.ts", @@ -36,7 +36,9 @@ "@types/xml2js": "catalog:" }, "dependencies": { + "@langchain/core": "catalog:", "@n8n/client-oauth2": "workspace:*", + "@n8n/config": "workspace:*", "aws4": "1.11.0", "axios": "catalog:", "concat-stream": "2.0.0", @@ -45,10 +47,10 @@ "file-type": "16.5.4", "form-data": "catalog:", "lodash": "catalog:", - "@langchain/core": "catalog:", "luxon": "catalog:", "mime-types": "2.1.35", "n8n-workflow": "workspace:*", + "nanoid": "catalog:", "oauth-1.0a": "2.2.6", "p-cancelable": "2.1.1", "pretty-bytes": "5.6.0", diff --git a/packages/core/src/DirectoryLoader.ts b/packages/core/src/DirectoryLoader.ts index a1401a8fb5..b0e77125a7 100644 --- a/packages/core/src/DirectoryLoader.ts +++ b/packages/core/src/DirectoryLoader.ts @@ -448,9 +448,9 @@ export class LazyPackageDirectoryLoader extends PackageDirectoryLoader { ); } - Logger.debug(`Lazy Loading credentials and nodes from ${this.packageJson.name}`, { - credentials: this.types.credentials?.length ?? 0, + Logger.debug(`Lazy-loading nodes and credentials from ${this.packageJson.name}`, { nodes: this.types.nodes?.length ?? 0, + credentials: this.types.credentials?.length ?? 0, }); this.isLazyLoaded = true; diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts index 17ccf15def..7d38f21184 100644 --- a/packages/core/src/InstanceSettings.ts +++ b/packages/core/src/InstanceSettings.ts @@ -1,9 +1,14 @@ import { createHash, randomBytes } from 'crypto'; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; -import { ApplicationError, jsonParse } from 'n8n-workflow'; +import { chmodSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'fs'; +import { ApplicationError, jsonParse, ALPHABET, toResult } from 'n8n-workflow'; +import { customAlphabet } from 'nanoid'; import path from 'path'; import { Service } from 'typedi'; +import { InstanceSettingsConfig } from './InstanceSettingsConfig'; + +const nanoid = customAlphabet(ALPHABET, 16); + interface ReadOnlySettings { encryptionKey: string; } @@ -38,17 +43,27 @@ export class InstanceSettings { private readonly settingsFile = path.join(this.n8nFolder, 'config'); + readonly enforceSettingsFilePermissions = this.loadEnforceSettingsFilePermissionsFlag(); + private settings = this.loadOrCreate(); + /** + * Fixed ID of this n8n instance, for telemetry. + * Derived from encryption key. Do not confuse with `hostId`. + * + * @example '258fce876abf5ea60eb86a2e777e5e190ff8f3e36b5b37aafec6636c31d4d1f9' + */ readonly instanceId = this.generateInstanceId(); readonly instanceType: InstanceType; - constructor() { + constructor(private readonly config: InstanceSettingsConfig) { const command = process.argv[2]; this.instanceType = ['webhook', 'worker'].includes(command) ? (command as InstanceType) : 'main'; + + this.hostId = `${this.instanceType}-${nanoid()}`; } /** @@ -61,6 +76,16 @@ export class InstanceSettings { */ instanceRole: InstanceRole = 'unset'; + /** + * Transient ID of this n8n instance, for scaling mode. + * Reset on restart. Do not confuse with `instanceId`. + * + * @example 'main-bnxa1riryKUNHtln' + * @example 'worker-nDJR0FnSd2Vf6DB5' + * @example 'webhook-jxQ7AO8IzxEtfW1F' + */ + readonly hostId: string; + get isLeader() { return this.instanceRole === 'leader'; } @@ -105,6 +130,7 @@ export class InstanceSettings { private loadOrCreate(): Settings { if (existsSync(this.settingsFile)) { const content = readFileSync(this.settingsFile, 'utf8'); + this.ensureSettingsFilePermissions(); const settings = jsonParse(content, { errorMessage: `Error parsing n8n-config file "${this.settingsFile}". It does not seem to be valid JSON.`, @@ -134,6 +160,7 @@ export class InstanceSettings { if (!inTest && !process.env.N8N_ENCRYPTION_KEY) { console.info(`No encryption key found - Auto-generated and saved to: ${this.settingsFile}`); } + this.ensureSettingsFilePermissions(); return settings; } @@ -147,6 +174,93 @@ export class InstanceSettings { private save(settings: Settings) { this.settings = settings; - writeFileSync(this.settingsFile, JSON.stringify(settings, null, '\t'), 'utf-8'); + writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, '\t'), { + mode: this.enforceSettingsFilePermissions.enforce ? 0o600 : undefined, + encoding: 'utf-8', + }); + } + + private loadEnforceSettingsFilePermissionsFlag(): { + isSet: boolean; + enforce: boolean; + } { + const { enforceSettingsFilePermissions } = this.config; + const isEnvVarSet = !!process.env.N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS; + if (this.isWindows()) { + if (isEnvVarSet) { + console.warn( + 'Ignoring N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS as it is not supported on Windows.', + ); + } + + return { + isSet: isEnvVarSet, + enforce: false, + }; + } + + return { + isSet: isEnvVarSet, + enforce: enforceSettingsFilePermissions, + }; + } + + /** + * Ensures that the settings file has the r/w permissions only for the owner. + */ + private ensureSettingsFilePermissions() { + // If the flag is explicitly set to false, skip the check + if (this.enforceSettingsFilePermissions.isSet && !this.enforceSettingsFilePermissions.enforce) { + return; + } + if (this.isWindows()) { + // Ignore windows as it does not support chmod. We have already logged a warning + return; + } + + const permissionsResult = toResult(() => { + const stats = statSync(this.settingsFile); + return stats.mode & 0o777; + }); + // If we can't determine the permissions, log a warning and skip the check + if (!permissionsResult.ok) { + console.warn( + `Could not ensure settings file permissions: ${permissionsResult.error.message}. To skip this check, set N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=false.`, + ); + return; + } + + const arePermissionsCorrect = permissionsResult.result === 0o600; + if (arePermissionsCorrect) { + return; + } + + // If the permissions are incorrect and the flag is not set, log a warning + if (!this.enforceSettingsFilePermissions.isSet) { + console.warn( + `Permissions 0${permissionsResult.result.toString(8)} for n8n settings file ${this.settingsFile} are too wide. This is ignored for now, but in the future n8n will attempt to change the permissions automatically. To automatically enforce correct permissions now set N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true (recommended), or turn this check off set N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=false.`, + ); + // The default is false so we skip the enforcement for now + return; + } + + if (this.enforceSettingsFilePermissions.enforce) { + console.warn( + `Permissions 0${permissionsResult.result.toString(8)} for n8n settings file ${this.settingsFile} are too wide. Changing permissions to 0600..`, + ); + const chmodResult = toResult(() => chmodSync(this.settingsFile, 0o600)); + if (!chmodResult.ok) { + // Some filesystems don't support permissions. In this case we log the + // error and ignore it. We might want to prevent the app startup in the + // future in this case. + console.warn( + `Could not enforce settings file permissions: ${chmodResult.error.message}. To skip this check, set N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=false.`, + ); + } + } + } + + private isWindows() { + return process.platform === 'win32'; } } diff --git a/packages/core/src/InstanceSettingsConfig.ts b/packages/core/src/InstanceSettingsConfig.ts new file mode 100644 index 0000000000..60baf8b80f --- /dev/null +++ b/packages/core/src/InstanceSettingsConfig.ts @@ -0,0 +1,12 @@ +import { Config, Env } from '@n8n/config'; + +@Config +export class InstanceSettingsConfig { + /** + * Whether to enforce that n8n settings file doesn't have overly wide permissions. + * If set to true, n8n will check the permissions of the settings file and + * attempt change them to 0600 (only owner has rw access) if they are too wide. + */ + @Env('N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS') + enforceSettingsFilePermissions: boolean = false; +} diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 6cbef1e1b8..10c44efced 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -110,6 +110,7 @@ import type { DeduplicationItemTypes, ICheckProcessedContextData, AiEvent, + ISupplyDataFunctions, } from 'n8n-workflow'; import { NodeConnectionType, @@ -2803,12 +2804,14 @@ async function getInputConnectionData( runExecutionData: IRunExecutionData, runIndex: number, connectionInputData: INodeExecutionData[], + inputData: ITaskDataConnections, additionalData: IWorkflowExecuteAdditionalData, - executeData: IExecuteData | undefined, + executeData: IExecuteData, mode: WorkflowExecuteMode, closeFunctions: CloseFunction[], inputName: NodeConnectionType, itemIndex: number, + abortSignal?: AbortSignal, ): Promise { const node = this.getNode(); const nodeType = workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion); @@ -2856,72 +2859,20 @@ async function getInputConnectionData( connectedNode.typeVersion, ); - const context = Object.assign({}, this); - - context.getNodeParameter = ( - parameterName: string, - itemIndex: number, - fallbackValue?: any, - options?: IGetNodeParameterOptions, - ) => { - return getNodeParameter( - workflow, - runExecutionData, - runIndex, - connectionInputData, - connectedNode, - parameterName, - itemIndex, - mode, - getAdditionalKeys(additionalData, mode, runExecutionData), - executeData, - fallbackValue, - { ...(options || {}), contextNode: node }, - ) as any; - }; - - // TODO: Check what else should be overwritten - context.getNode = () => { - return deepCopy(connectedNode); - }; - - context.getCredentials = async (key: string) => { - try { - return await getCredentials( - workflow, - connectedNode, - key, - additionalData, - mode, - executeData, - runExecutionData, - runIndex, - connectionInputData, - itemIndex, - ); - } catch (error) { - // Display the error on the node which is causing it - - let currentNodeRunIndex = 0; - if (runExecutionData.resultData.runData.hasOwnProperty(node.name)) { - currentNodeRunIndex = runExecutionData.resultData.runData[node.name].length; - } - - await addExecutionDataFunctions( - 'input', - connectedNode.name, - error, - runExecutionData, - inputName, - additionalData, - node.name, - runIndex, - currentNodeRunIndex, - ); - - throw error; - } - }; + // eslint-disable-next-line @typescript-eslint/no-use-before-define + const context = getSupplyDataFunctions( + workflow, + runExecutionData, + runIndex, + connectionInputData, + inputData, + connectedNode, + additionalData, + executeData, + mode, + closeFunctions, + abortSignal, + ); if (!nodeType.supplyData) { if (nodeType.description.outputs.includes(NodeConnectionType.AiTool)) { @@ -3758,7 +3709,7 @@ export function getExecuteFunctions( continueOnFail: () => { return continueOnFail(node); }, - evaluateExpression: (expression: string, itemIndex: number) => { + evaluateExpression(expression: string, itemIndex: number) { return workflow.expression.resolveSimpleParameterValue( `=${expression}`, {}, @@ -3808,12 +3759,14 @@ export function getExecuteFunctions( runExecutionData, runIndex, connectionInputData, + inputData, additionalData, executeData, mode, closeFunctions, inputName, itemIndex, + abortSignal, ); }, @@ -4027,7 +3980,7 @@ export function getExecuteFunctions( constructExecutionMetaData, }, nodeHelpers: getNodeHelperFunctions(additionalData, workflow.id), - logAiEvent: async (eventName: AiEvent, msg: string) => { + logAiEvent: (eventName: AiEvent, msg: string) => { return additionalData.logAiEvent(eventName, { executionId: additionalData.executionId ?? 'unsaved-execution', nodeName: node.name, @@ -4055,6 +4008,270 @@ export function getExecuteFunctions( })(workflow, runExecutionData, connectionInputData, inputData, node) as IExecuteFunctions; } +export function getSupplyDataFunctions( + workflow: Workflow, + runExecutionData: IRunExecutionData, + runIndex: number, + connectionInputData: INodeExecutionData[], + inputData: ITaskDataConnections, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + executeData: IExecuteData, + mode: WorkflowExecuteMode, + closeFunctions: CloseFunction[], + abortSignal?: AbortSignal, +): ISupplyDataFunctions { + return { + ...getCommonWorkflowFunctions(workflow, node, additionalData), + ...executionCancellationFunctions(abortSignal), + getMode: () => mode, + getCredentials: async (type, itemIndex) => + await getCredentials( + workflow, + node, + type, + additionalData, + mode, + executeData, + runExecutionData, + runIndex, + connectionInputData, + itemIndex, + ), + continueOnFail: () => continueOnFail(node), + evaluateExpression: (expression: string, itemIndex: number) => + workflow.expression.resolveSimpleParameterValue( + `=${expression}`, + {}, + runExecutionData, + runIndex, + itemIndex, + node.name, + connectionInputData, + mode, + getAdditionalKeys(additionalData, mode, runExecutionData), + executeData, + ), + executeWorkflow: async ( + workflowInfo: IExecuteWorkflowInfo, + inputData?: INodeExecutionData[], + parentCallbackManager?: CallbackManager, + ) => + await additionalData + .executeWorkflow(workflowInfo, additionalData, { + parentWorkflowId: workflow.id?.toString(), + inputData, + parentWorkflowSettings: workflow.settings, + node, + parentCallbackManager, + }) + .then( + async (result) => + await Container.get(BinaryDataService).duplicateBinaryData( + workflow.id, + additionalData.executionId!, + result, + ), + ), + getNodeOutputs() { + const nodeType = workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion); + return NodeHelpers.getNodeOutputs(workflow, node, nodeType.description).map((output) => { + if (typeof output === 'string') { + return { + type: output, + }; + } + return output; + }); + }, + async getInputConnectionData( + inputName: NodeConnectionType, + itemIndex: number, + ): Promise { + return await getInputConnectionData.call( + this, + workflow, + runExecutionData, + runIndex, + connectionInputData, + inputData, + additionalData, + executeData, + mode, + closeFunctions, + inputName, + itemIndex, + abortSignal, + ); + }, + getInputData: (inputIndex = 0, inputName = 'main') => { + if (!inputData.hasOwnProperty(inputName)) { + // Return empty array because else it would throw error when nothing is connected to input + return []; + } + + // TODO: Check if nodeType has input with that index defined + if (inputData[inputName].length < inputIndex) { + throw new ApplicationError('Could not get input with given index', { + extra: { inputIndex, inputName }, + }); + } + + if (inputData[inputName][inputIndex] === null) { + throw new ApplicationError('Value of input was not set', { + extra: { inputIndex, inputName }, + }); + } + + return inputData[inputName][inputIndex]; + }, + getNodeParameter: (( + parameterName: string, + itemIndex: number, + fallbackValue?: any, + options?: IGetNodeParameterOptions, + ) => + getNodeParameter( + workflow, + runExecutionData, + runIndex, + connectionInputData, + node, + parameterName, + itemIndex, + mode, + getAdditionalKeys(additionalData, mode, runExecutionData), + executeData, + fallbackValue, + options, + )) as ISupplyDataFunctions['getNodeParameter'], + getWorkflowDataProxy: (itemIndex: number) => + new WorkflowDataProxy( + workflow, + runExecutionData, + runIndex, + itemIndex, + node.name, + connectionInputData, + {}, + mode, + getAdditionalKeys(additionalData, mode, runExecutionData), + executeData, + ).getDataProxy(), + sendMessageToUI(...args: any[]): void { + if (mode !== 'manual') { + return; + } + try { + if (additionalData.sendDataToUI) { + args = args.map((arg) => { + // prevent invalid dates from being logged as null + if (arg.isLuxonDateTime && arg.invalidReason) return { ...arg }; + + // log valid dates in human readable format, as in browser + if (arg.isLuxonDateTime) return new Date(arg.ts).toString(); + if (arg instanceof Date) return arg.toString(); + + return arg; + }); + + additionalData.sendDataToUI('sendConsoleMessage', { + source: `[Node: "${node.name}"]`, + messages: args, + }); + } + } catch (error) { + Logger.warn(`There was a problem sending message to UI: ${error.message}`); + } + }, + logAiEvent: (eventName: AiEvent, msg: string) => + additionalData.logAiEvent(eventName, { + executionId: additionalData.executionId ?? 'unsaved-execution', + nodeName: node.name, + workflowName: workflow.name ?? 'Unnamed workflow', + nodeType: node.type, + workflowId: workflow.id ?? 'unsaved-workflow', + msg, + }), + addInputData( + connectionType: NodeConnectionType, + data: INodeExecutionData[][], + ): { index: number } { + const nodeName = this.getNode().name; + let currentNodeRunIndex = 0; + if (runExecutionData.resultData.runData.hasOwnProperty(nodeName)) { + currentNodeRunIndex = runExecutionData.resultData.runData[nodeName].length; + } + + addExecutionDataFunctions( + 'input', + this.getNode().name, + data, + runExecutionData, + connectionType, + additionalData, + node.name, + runIndex, + currentNodeRunIndex, + ).catch((error) => { + Logger.warn( + `There was a problem logging input data of node "${this.getNode().name}": ${ + error.message + }`, + ); + }); + + return { index: currentNodeRunIndex }; + }, + addOutputData( + connectionType: NodeConnectionType, + currentNodeRunIndex: number, + data: INodeExecutionData[][], + ): void { + addExecutionDataFunctions( + 'output', + this.getNode().name, + data, + runExecutionData, + connectionType, + additionalData, + node.name, + runIndex, + currentNodeRunIndex, + ).catch((error) => { + Logger.warn( + `There was a problem logging output data of node "${this.getNode().name}": ${ + error.message + }`, + ); + }); + }, + helpers: { + createDeferredPromise, + copyInputItems, + ...getRequestHelperFunctions( + workflow, + node, + additionalData, + runExecutionData, + connectionInputData, + ), + ...getSSHTunnelFunctions(), + ...getFileSystemHelperFunctions(node), + ...getBinaryHelperFunctions(additionalData, workflow.id), + ...getCheckProcessedHelperFunctions(workflow, node), + assertBinaryData: (itemIndex, propertyName) => + assertBinaryData(inputData, node, itemIndex, propertyName, 0), + getBinaryDataBuffer: async (itemIndex, propertyName) => + await getBinaryDataBuffer(inputData, itemIndex, propertyName, 0), + + returnJsonArray, + normalizeItems, + constructExecutionMetaData, + }, + }; +} + /** * Returns the execute functions regular nodes have access to when single-function is defined. */ @@ -4197,7 +4414,7 @@ export function getExecuteSingleFunctions( getBinaryDataBuffer: async (propertyName, inputIndex = 0) => await getBinaryDataBuffer(inputData, itemIndex, propertyName, inputIndex), }, - logAiEvent: async (eventName: AiEvent, msg: string) => { + logAiEvent: (eventName: AiEvent, msg: string) => { return additionalData.logAiEvent(eventName, { executionId: additionalData.executionId ?? 'unsaved-execution', nodeName: node.name, @@ -4427,6 +4644,7 @@ export function getExecuteWebhookFunctions( runExecutionData, runIndex, connectionInputData, + {} as ITaskDataConnections, additionalData, executeData, mode, @@ -4436,6 +4654,36 @@ export function getExecuteWebhookFunctions( ); }, getMode: () => mode, + evaluateExpression: (expression: string, evaluateItemIndex?: number) => { + const itemIndex = evaluateItemIndex === undefined ? 0 : evaluateItemIndex; + const runIndex = 0; + + let connectionInputData: INodeExecutionData[] = []; + let executionData: IExecuteData | undefined; + + if (runExecutionData?.executionData !== undefined) { + executionData = runExecutionData.executionData.nodeExecutionStack[0]; + + if (executionData !== undefined) { + connectionInputData = executionData.data.main[0]!; + } + } + + const additionalKeys = getAdditionalKeys(additionalData, mode, runExecutionData); + + return workflow.expression.resolveSimpleParameterValue( + `=${expression}`, + {}, + runExecutionData, + runIndex, + itemIndex, + node.name, + connectionInputData, + mode, + additionalKeys, + executionData, + ); + }, getNodeParameter: ( parameterName: string, fallbackValue?: any, diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 606f624d02..6f8b43a660 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -286,6 +286,149 @@ export class DirectedGraph { ); } + /** + * Returns all strongly connected components. + * + * Strongly connected components are a set of nodes where it's possible to + * reach every node from every node. + * + * Strongly connected components are mutually exclusive in directed graphs, + * e.g. they cannot overlap. + * + * The smallest strongly connected component is a single node, since it can + * reach itself from itself by not following any edges. + * + * The algorithm implement here is Tarjan's algorithm. + * + * Example: + * ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ + * │node1├────►node2◄────┤node3├────►node5│ + * └─────┘ └──┬──┘ └──▲──┘ └▲───┬┘ + * │ │ │ │ + * ┌──▼──┐ │ ┌┴───▼┐ + * │node4├───────┘ │node6│ + * └─────┘ └─────┘ + * + * The strongly connected components are + * 1. node1 + * 2. node2, node4, node3 + * 3. node5, node6 + * + * Further reading: + * https://en.wikipedia.org/wiki/Strongly_connected_component + * https://www.youtube.com/watch?v=wUgWX0nc4NY + */ + getStronglyConnectedComponents(): Array> { + let id = 0; + const visited = new Set(); + const ids = new Map(); + const lowLinkValues = new Map(); + const stack: INode[] = []; + const stronglyConnectedComponents: Array> = []; + + const followNode = (node: INode) => { + if (visited.has(node)) { + return; + } + + visited.add(node); + lowLinkValues.set(node, id); + ids.set(node, id); + id++; + stack.push(node); + + const directChildren = this.getDirectChildConnections(node).map((c) => c.to); + for (const child of directChildren) { + followNode(child); + + // if node is on stack min the low id + if (stack.includes(child)) { + const childLowLinkValue = lowLinkValues.get(child); + const ownLowLinkValue = lowLinkValues.get(node); + a.ok(childLowLinkValue !== undefined); + a.ok(ownLowLinkValue !== undefined); + const lowestLowLinkValue = Math.min(childLowLinkValue, ownLowLinkValue); + + lowLinkValues.set(node, lowestLowLinkValue); + } + } + + // after we visited all children, check if the low id is the same as the + // nodes id, which means we found a strongly connected component + const ownId = ids.get(node); + const ownLowLinkValue = lowLinkValues.get(node); + a.ok(ownId !== undefined); + a.ok(ownLowLinkValue !== undefined); + + if (ownId === ownLowLinkValue) { + // pop from the stack until the stack is empty or we find a node that + // has a different low id + const scc: Set = new Set(); + let next = stack.at(-1); + + while (next && lowLinkValues.get(next) === ownId) { + stack.pop(); + scc.add(next); + next = stack.at(-1); + } + + if (scc.size > 0) { + stronglyConnectedComponents.push(scc); + } + } + }; + + for (const node of this.nodes.values()) { + followNode(node); + } + + return stronglyConnectedComponents; + } + + private depthFirstSearchRecursive( + from: INode, + fn: (node: INode) => boolean, + seen: Set, + ): INode | undefined { + if (seen.has(from)) { + return undefined; + } + seen.add(from); + + if (fn(from)) { + return from; + } + + for (const childConnection of this.getDirectChildConnections(from)) { + const found = this.depthFirstSearchRecursive(childConnection.to, fn, seen); + + if (found) { + return found; + } + } + + return undefined; + } + + /** + * Like `Array.prototype.find` but for directed graphs. + * + * Starting from, and including, the `from` node this calls the provided + * predicate function with every child node until the predicate function + * returns true. + * + * The search is depth first, meaning every branch is exhausted before the + * next branch is tried. + * + * The first node for which the predicate function returns true is returned. + * + * If the graph is exhausted and the predicate function never returned true, + * undefined is returned instead. + */ + depthFirstSearch({ from, fn }: { from: INode; fn: (node: INode) => boolean }): INode | undefined { + return this.depthFirstSearchRecursive(from, fn, new Set()); + } + toWorkflow(parameters: Omit): Workflow { return new Workflow({ ...parameters, diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts index d6eedf416d..9530ed2217 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts @@ -9,6 +9,7 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data +import type { INode } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; import { createNodeData, defaultWorkflowParameter } from './helpers'; @@ -89,6 +90,115 @@ describe('DirectedGraph', () => { }); }); + describe('getStronglyConnectedComponents', () => { + // ┌─────┐ ┌─────┐ ┌─────┐ + // │node1├───►│node2├───►│node4│ + // └─────┘ └──┬──┘ └─────┘ + // ▲ │ + // │ │ + // ┌──┴──┐ │ + // │node3│◄──────┘ + // └─────┘ + test('find strongly connected components', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + const node4 = createNodeData({ name: 'Node4' }); + const graph = new DirectedGraph() + .addNodes(node1, node2, node3, node4) + .addConnections( + { from: node1, to: node2 }, + { from: node2, to: node3 }, + { from: node3, to: node1 }, + { from: node2, to: node4 }, + ); + + // ACT + const stronglyConnectedComponents = graph.getStronglyConnectedComponents(); + + // ASSERT + expect(stronglyConnectedComponents).toHaveLength(2); + expect(stronglyConnectedComponents).toContainEqual(new Set([node4])); + expect(stronglyConnectedComponents).toContainEqual(new Set([node3, node2, node1])); + }); + + // ┌────┐ + // ┌───────┐ │ ├─ + // │trigger├──┬──►loop│ + // └───────┘ │ │ ├────┐ + // │ └────┘ │ + // └─────────┐ │ + // ┌────┐ │ │ + // ┌───►node├─┘ │ + // │ └────┘ │ + // │ │ + // └─────────────┘ + test('find strongly connected components even if they use different output indexes', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const loop = createNodeData({ name: 'loop' }); + const node = createNodeData({ name: 'node' }); + const graph = new DirectedGraph() + .addNodes(trigger, loop, node) + .addConnections( + { from: trigger, to: loop }, + { from: loop, outputIndex: 1, to: node }, + { from: node, to: loop }, + ); + + // ACT + const stronglyConnectedComponents = graph.getStronglyConnectedComponents(); + + // ASSERT + expect(stronglyConnectedComponents).toHaveLength(2); + expect(stronglyConnectedComponents).toContainEqual(new Set([trigger])); + expect(stronglyConnectedComponents).toContainEqual(new Set([node, loop])); + }); + }); + + describe('depthFirstSearch', () => { + // ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ + // │node0├───►│node1├───►│node2├───►│node4│───►│node5│ + // └─────┘ └─────┘ └──┬──┘ └─────┘ └─────┘ + // ▲ │ + // │ │ + // ┌──┴──┐ │ + // │node3│◄──────┘ + // └─────┘ + test('calls nodes in the correct order and stops when it found the node', () => { + // ARRANGE + const node0 = createNodeData({ name: 'Node0' }); + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + const node4 = createNodeData({ name: 'Node4' }); + const node5 = createNodeData({ name: 'Node5' }); + const graph = new DirectedGraph() + .addNodes(node0, node1, node2, node3, node4, node5) + .addConnections( + { from: node0, to: node1 }, + { from: node1, to: node2 }, + { from: node2, to: node3 }, + { from: node3, to: node1 }, + { from: node2, to: node4 }, + { from: node4, to: node5 }, + ); + const fn = jest.fn().mockImplementation((node: INode) => node === node4); + + // ACT + const foundNode = graph.depthFirstSearch({ + from: node0, + fn, + }); + + // ASSERT + expect(foundNode).toBe(node4); + expect(fn).toHaveBeenCalledTimes(5); + expect(fn.mock.calls).toEqual([[node0], [node1], [node2], [node3], [node4]]); + }); + }); + describe('getParentConnections', () => { // ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ // │node1├──►│node2├──►│node3│──►│node4│ diff --git a/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts index bf37ec7636..5daea46ef6 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts @@ -23,7 +23,7 @@ describe('cleanRunData', () => { }; // ACT - const newRunData = cleanRunData(runData, graph, [node1]); + const newRunData = cleanRunData(runData, graph, new Set([node1])); // ASSERT expect(newRunData).toEqual({}); @@ -47,7 +47,7 @@ describe('cleanRunData', () => { }; // ACT - const newRunData = cleanRunData(runData, graph, [node2]); + const newRunData = cleanRunData(runData, graph, new Set([node2])); // ASSERT expect(newRunData).toEqual({ [node1.name]: runData[node1.name] }); @@ -78,7 +78,7 @@ describe('cleanRunData', () => { }; // ACT - const newRunData = cleanRunData(runData, graph, [node2]); + const newRunData = cleanRunData(runData, graph, new Set([node2])); // ASSERT // TODO: Find out if this is a desirable result in milestone 2 diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts index 57022d862c..ab33ccf8ed 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts @@ -48,8 +48,8 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger: node, destination: node }); - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node); }); // ►► @@ -67,8 +67,8 @@ describe('findStartNodes', () => { { const startNodes = findStartNodes({ graph, trigger, destination }); - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(trigger); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(trigger); } // if the trigger has run data @@ -79,8 +79,8 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger, destination, runData }); - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(destination); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(destination); } }); @@ -115,8 +115,8 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger, destination: node, runData }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node); }); // ┌─────┐ ┌─────┐ ►► @@ -156,9 +156,9 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger, destination: node4 }); // ASSERT - expect(startNodes).toHaveLength(1); + expect(startNodes.size).toBe(1); // no run data means the trigger is the start node - expect(startNodes[0]).toEqual(trigger); + expect(startNodes).toContainEqual(trigger); } { @@ -175,8 +175,8 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger, destination: node4, runData }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node4); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node4); } }); @@ -211,8 +211,8 @@ describe('findStartNodes', () => { }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node); }); // ►► @@ -246,8 +246,8 @@ describe('findStartNodes', () => { }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node); }); // ►► @@ -286,8 +286,8 @@ describe('findStartNodes', () => { }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node); }); // ►► @@ -324,8 +324,8 @@ describe('findStartNodes', () => { }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node3); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node3); }); // ►► @@ -360,8 +360,8 @@ describe('findStartNodes', () => { }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node2); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node2); }); // ►► @@ -392,7 +392,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes({ graph, trigger, destination: node2, runData, pinData }); // ASSERT - expect(startNodes).toHaveLength(1); - expect(startNodes[0]).toEqual(node2); + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(node2); }); }); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts index dffbe310d1..d8c3485d65 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts @@ -52,15 +52,15 @@ describe('getSourceDataGroups', () => { expect(groups).toHaveLength(2); const group1 = groups[0]; - expect(group1).toHaveLength(2); - expect(group1[0]).toEqual({ + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ from: source1, outputIndex: 0, type: NodeConnectionType.Main, inputIndex: 0, to: node, }); - expect(group1[1]).toEqual({ + expect(group1.connections[1]).toEqual({ from: source3, outputIndex: 0, type: NodeConnectionType.Main, @@ -69,8 +69,8 @@ describe('getSourceDataGroups', () => { }); const group2 = groups[1]; - expect(group2).toHaveLength(1); - expect(group2[0]).toEqual({ + expect(group2.connections).toHaveLength(1); + expect(group2.connections[0]).toEqual({ from: source2, outputIndex: 0, type: NodeConnectionType.Main, @@ -116,15 +116,15 @@ describe('getSourceDataGroups', () => { expect(groups).toHaveLength(2); const group1 = groups[0]; - expect(group1).toHaveLength(2); - expect(group1[0]).toEqual({ + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ from: source1, outputIndex: 0, type: NodeConnectionType.Main, inputIndex: 0, to: node, }); - expect(group1[1]).toEqual({ + expect(group1.connections[1]).toEqual({ from: source3, outputIndex: 0, type: NodeConnectionType.Main, @@ -133,8 +133,8 @@ describe('getSourceDataGroups', () => { }); const group2 = groups[1]; - expect(group2).toHaveLength(1); - expect(group2[0]).toEqual({ + expect(group2.connections).toHaveLength(1); + expect(group2.connections[0]).toEqual({ from: source2, outputIndex: 0, type: NodeConnectionType.Main, @@ -152,7 +152,7 @@ describe('getSourceDataGroups', () => { //┌───────┐1 │ └────┘ //│source3├────┘ //└───────┘ - it('groups sources into possibly complete sets if all of them have data', () => { + it('groups sources into one complete set with 2 connections and one incomplete set with 1 connection', () => { // ARRANGE const source1 = createNodeData({ name: 'source1' }); const source2 = createNodeData({ name: 'source2' }); @@ -176,23 +176,341 @@ describe('getSourceDataGroups', () => { const groups = getSourceDataGroups(graph, node, runData, pinnedData); // ASSERT - expect(groups).toHaveLength(1); + const completeGroups = groups.filter((g) => g.complete); + { + expect(completeGroups).toHaveLength(1); + const group1 = completeGroups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1.connections[1]).toEqual({ + from: source3, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + } - const group1 = groups[0]; - expect(group1).toHaveLength(2); - expect(group1[0]).toEqual({ - from: source2, + const incompleteGroups = groups.filter((g) => !g.complete); + { + expect(incompleteGroups).toHaveLength(1); + const group1 = incompleteGroups[0]; + expect(group1.connections).toHaveLength(1); + expect(group1.connections[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + } + }); + + //┌───────┐0 + //│source1├───────┐ + //└───────┘ │ + // │ + //┌───────┐1 │ + //│source2├───────┤ ┌────┐ + //└───────┘ └────► │ + // │node│ + //┌───────┐1 ┌────► │ + //│source3├───────┤ └────┘ + //└───────┘ │ + // │ + //┌───────┐0 │ + //│source4├───────┘ + //└───────┘ + it('groups sources into one complete set with 2 connections and one incomplete set with 2 connection', () => { + // ARRANGE + const source1 = createNodeData({ name: 'source1' }); + const source2 = createNodeData({ name: 'source2' }); + const source3 = createNodeData({ name: 'source3' }); + const source4 = createNodeData({ name: 'source4' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, source2, source3, source4, node) + .addConnections( + { from: source1, to: node, inputIndex: 0 }, + { from: source2, to: node, inputIndex: 0 }, + { from: source3, to: node, inputIndex: 1 }, + { from: source4, to: node, inputIndex: 1 }, + ); + const runData: IRunData = { + [source2.name]: [toITaskData([{ data: { value: 1 } }])], + [source3.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinnedData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + // ASSERT + const completeGroups = groups.filter((g) => g.complete); + { + expect(completeGroups).toHaveLength(1); + const group1 = completeGroups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1.connections[1]).toEqual({ + from: source3, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + } + + const incompleteGroups = groups.filter((g) => !g.complete); + { + expect(incompleteGroups).toHaveLength(1); + const group1 = incompleteGroups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1.connections[1]).toEqual({ + from: source4, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + } + }); + + // ┌───────┐1 + // │source1├───────┐ + // └───────┘ │ + // │ + // ┌───────┐0 │ + // │source2├───────┤ ┌────┐ + // └───────┘ └────► │ + // │node│ + // ┌───────┐0 ┌────► │ + // │source3├───────┘ └────┘ + // └───────┘ + it('groups sources into two incomplete sets, one with 1 connection without and one with 2 connections one with data and one without', () => { + // ARRANGE + const source1 = createNodeData({ name: 'source1' }); + const source2 = createNodeData({ name: 'source2' }); + const source3 = createNodeData({ name: 'source3' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, source2, source3, node) + .addConnections( + { from: source1, to: node, inputIndex: 0 }, + { from: source2, to: node, inputIndex: 0 }, + { from: source3, to: node, inputIndex: 1 }, + ); + const runData: IRunData = { + [source1.name]: [toITaskData([{ data: { node: 'source1' } }])], + }; + const pinnedData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + // ASSERT + const completeGroups = groups.filter((g) => g.complete); + expect(completeGroups).toHaveLength(0); + + const incompleteGroups = groups.filter((g) => !g.complete); + expect(incompleteGroups).toHaveLength(2); + + const group1 = incompleteGroups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.connections[0]).toEqual({ + from: source1, outputIndex: 0, type: NodeConnectionType.Main, inputIndex: 0, to: node, }); - expect(group1[1]).toEqual({ + expect(group1.connections[1]).toEqual({ from: source3, outputIndex: 0, type: NodeConnectionType.Main, inputIndex: 1, to: node, }); + + const group2 = incompleteGroups[1]; + expect(group2.connections).toHaveLength(1); + expect(group2.connections[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + }); + + // ┌─────┐1 ►► + // ┌─►│Node1┼──┐ ┌─────┐ + // ┌───────┐1│ └─────┘ └──►│ │ + // │Trigger├─┤ │Node3│ + // └───────┘ │ ┌─────┐0 ┌──►│ │ + // └─►│Node2├──┘ └─────┘ + // └─────┘ + test('return an incomplete group when there is no data on input 2', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, node3) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: node1, to: node3, inputIndex: 0 }, + { from: node2, to: node3, inputIndex: 1 }, + ); + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { nodeName: 'trigger' } }])], + [node1.name]: [toITaskData([{ data: { nodeName: 'node1' } }])], + }; + const pinData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node3, runData, pinData); + + // ASSERT + expect(groups).toHaveLength(1); + const group1 = groups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.complete).toEqual(false); + }); + + // ┌─────┐0 ►► + // ┌─►│Node1┼──┐ ┌─────┐ + // ┌───────┐1│ └─────┘ └──►│ │ + // │Trigger├─┤ │Node3│ + // └───────┘ │ ┌─────┐1 ┌──►│ │ + // └─►│Node2├──┘ └─────┘ + // └─────┘ + test('return an incomplete group when there is no data on input 1', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, node3) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: node1, to: node3, inputIndex: 0 }, + { from: node2, to: node3, inputIndex: 1 }, + ); + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { nodeName: 'trigger' } }])], + [node2.name]: [toITaskData([{ data: { nodeName: 'node2' } }])], + }; + const pinData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node3, runData, pinData); + + // ASSERT + expect(groups).toHaveLength(1); + const group1 = groups[0]; + expect(group1.connections).toHaveLength(2); + expect(group1.complete).toEqual(false); + }); + + it('terminates with negative input indexes', () => { + // ARRANGE + const source1 = createNodeData({ name: 'source1' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, node) + .addConnections({ from: source1, to: node, inputIndex: -1 }); + const runData: IRunData = { + [source1.name]: [toITaskData([{ data: { node: source1.name } }])], + }; + const pinnedData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + // ASSERT + expect(groups).toHaveLength(1); + const group1 = groups[0]; + expect(group1.connections).toHaveLength(1); + expect(group1.connections[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: -1, + to: node, + }); + }); + + it('terminates inputs with missing connections', () => { + // ARRANGE + const source1 = createNodeData({ name: 'source1' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, node) + .addConnections({ from: source1, to: node, inputIndex: 1 }); + const runData: IRunData = { + [source1.name]: [toITaskData([{ data: { node: source1.name } }])], + }; + const pinnedData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + // ASSERT + expect(groups).toHaveLength(1); + const group1 = groups[0]; + expect(group1.connections).toHaveLength(1); + expect(group1.connections[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + }); + + it('terminates if the graph has no connections', () => { + // ARRANGE + const source1 = createNodeData({ name: 'source1' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph().addNodes(source1, node); + const runData: IRunData = { + [source1.name]: [toITaskData([{ data: { node: source1.name } }])], + }; + const pinnedData: IPinData = {}; + + // ACT + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + // ASSERT + expect(groups).toHaveLength(0); }); }); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts new file mode 100644 index 0000000000..def9fed0ff --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts @@ -0,0 +1,116 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to +// XX denotes that the node is disabled +// PD denotes that the node has pinned data + +import { createNodeData } from './helpers'; +import { DirectedGraph } from '../DirectedGraph'; +import { handleCycles } from '../handleCycles'; + +describe('handleCycles', () => { + // ┌────┐ ┌─────────┐ + //┌───────┐ │ ├──────────►afterLoop│ + //│trigger├────┬───►loop│ └─────────┘ + //└───────┘ │ │ ├─┐ ►► + // │ └────┘ │ ┌──────┐ + // │ └───►inLoop├────┐ + // │ └──────┘ │ + // │ │ + // └──────────────────────────┘ + test('if the start node is within a cycle it returns the start of the cycle as the new start node', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const loop = createNodeData({ name: 'loop' }); + const inLoop = createNodeData({ name: 'inLoop' }); + const afterLoop = createNodeData({ name: 'afterLoop' }); + const graph = new DirectedGraph() + .addNodes(trigger, loop, inLoop, afterLoop) + .addConnections( + { from: trigger, to: loop }, + { from: loop, outputIndex: 0, to: afterLoop }, + { from: loop, outputIndex: 1, to: inLoop }, + { from: inLoop, to: loop }, + ); + const startNodes = new Set([inLoop]); + + // ACT + const newStartNodes = handleCycles(graph, startNodes, trigger); + + // ASSERT + expect(newStartNodes.size).toBe(1); + expect(newStartNodes).toContainEqual(loop); + }); + + // ┌────┐ ┌─────────┐ + //┌───────┐ │ ├──────────►afterLoop│ + //│trigger├────┬───►loop│ └─────────┘ + //└───────┘ │ │ ├─┐ ►► + // │ └────┘ │ ┌──────┐ + // │ └───►inLoop├────┐ + // │ └──────┘ │ + // │ │ + // └──────────────────────────┘ + test('does not mutate `startNodes`', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const loop = createNodeData({ name: 'loop' }); + const inLoop = createNodeData({ name: 'inLoop' }); + const afterLoop = createNodeData({ name: 'afterLoop' }); + const graph = new DirectedGraph() + .addNodes(trigger, loop, inLoop, afterLoop) + .addConnections( + { from: trigger, to: loop }, + { from: loop, outputIndex: 0, to: afterLoop }, + { from: loop, outputIndex: 1, to: inLoop }, + { from: inLoop, to: loop }, + ); + const startNodes = new Set([inLoop]); + + // ACT + handleCycles(graph, startNodes, trigger); + + // ASSERT + expect(startNodes.size).toBe(1); + expect(startNodes).toContainEqual(inLoop); + }); + + // ►► + // ┌────┐ ┌─────────┐ + //┌───────┐ │ ├──────────►afterLoop│ + //│trigger├────┬───►loop│ └─────────┘ + //└───────┘ │ │ ├─┐ + // │ └────┘ │ ┌──────┐ + // │ └───►inLoop├────┐ + // │ └──────┘ │ + // │ │ + // └──────────────────────────┘ + test('if the start node is not within a cycle it returns the same node as the new start node', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const loop = createNodeData({ name: 'loop' }); + const inLoop = createNodeData({ name: 'inLoop' }); + const afterLoop = createNodeData({ name: 'afterLoop' }); + const graph = new DirectedGraph() + .addNodes(trigger, loop, inLoop, afterLoop) + .addConnections( + { from: trigger, to: loop }, + { from: loop, outputIndex: 0, to: afterLoop }, + { from: loop, outputIndex: 1, to: inLoop }, + { from: inLoop, to: loop }, + ); + const startNodes = new Set([afterLoop]); + + // ACT + const newStartNodes = handleCycles(graph, startNodes, trigger); + + // ASSERT + expect(newStartNodes.size).toBe(1); + expect(newStartNodes).toContainEqual(afterLoop); + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index a4bcac23a5..b78b9df135 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -10,9 +10,19 @@ // PD denotes that the node has pinned data import { AssertionError } from 'assert'; -import { type IPinData, type IRunData } from 'n8n-workflow'; +import type { + INodeExecutionData, + ISourceData, + IWaitingForExecution, + IWaitingForExecutionSource, +} from 'n8n-workflow'; +import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; -import { recreateNodeExecutionStack } from '@/PartialExecutionUtils/recreateNodeExecutionStack'; +import { + addWaitingExecution, + addWaitingExecutionSource, + recreateNodeExecutionStack, +} from '@/PartialExecutionUtils/recreateNodeExecutionStack'; import { createNodeData, toITaskData } from './helpers'; import { DirectedGraph } from '../DirectedGraph'; @@ -33,7 +43,7 @@ describe('recreateNodeExecutionStack', () => { .addConnections({ from: trigger, to: node }); const workflow = findSubgraph({ graph, destination: node, trigger }); - const startNodes = [node]; + const startNodes = new Set([node]); const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], }; @@ -41,7 +51,7 @@ describe('recreateNodeExecutionStack', () => { // ACT const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, runData, pinData); // ASSERT expect(nodeExecutionStack).toHaveLength(1); @@ -62,17 +72,8 @@ describe('recreateNodeExecutionStack', () => { }, }, ]); - - expect(waitingExecution).toEqual({ node: { '0': { main: [[{ json: { value: 1 } }]] } } }); - expect(waitingExecutionSource).toEqual({ - node: { - '0': { - main: [ - { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, - ], - }, - }, - }); + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); }); // ►► @@ -87,13 +88,13 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const startNodes = [trigger]; + const startNodes = new Set([trigger]); const runData: IRunData = {}; const pinData: IPinData = {}; // ACT const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, runData, pinData); // ASSERT expect(nodeExecutionStack).toHaveLength(1); @@ -105,8 +106,8 @@ describe('recreateNodeExecutionStack', () => { }, ]); - expect(waitingExecution).toEqual({ node: { '0': { main: [null] } } }); - expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); }); // PinData ►► @@ -121,7 +122,7 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const startNodes = [node]; + const startNodes = new Set([node]); const runData: IRunData = {}; const pinData: IPinData = { [trigger.name]: [{ json: { value: 1 } }], @@ -129,7 +130,7 @@ describe('recreateNodeExecutionStack', () => { // ACT const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, runData, pinData); // ASSERT expect(nodeExecutionStack).toHaveLength(1); @@ -151,8 +152,8 @@ describe('recreateNodeExecutionStack', () => { }, ]); - expect(waitingExecution).toEqual({ node: { '0': { main: [null] } } }); - expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); }); // XX ►► @@ -169,16 +170,16 @@ describe('recreateNodeExecutionStack', () => { .addNodes(trigger, node1, node2) .addConnections({ from: trigger, to: node1 }, { from: node1, to: node2 }); - const startNodes = [node2]; + const startNodes = new Set([node2]); const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], }; const pinData = {}; // ACT & ASSERT - expect(() => - recreateNodeExecutionStack(graph, startNodes, node2, runData, pinData), - ).toThrowError(AssertionError); + expect(() => recreateNodeExecutionStack(graph, startNodes, runData, pinData)).toThrowError( + AssertionError, + ); }); // ►► @@ -204,7 +205,7 @@ describe('recreateNodeExecutionStack', () => { { from: node2, to: node3 }, ); - const startNodes = [node3]; + const startNodes = new Set([node3]); const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], [node1.name]: [toITaskData([{ data: { value: 1 } }])], @@ -214,10 +215,9 @@ describe('recreateNodeExecutionStack', () => { // ACT const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); + recreateNodeExecutionStack(graph, startNodes, runData, pinData); // ASSERT - expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: { value: 1 } }]] }, @@ -251,19 +251,8 @@ describe('recreateNodeExecutionStack', () => { }, ]); - expect(waitingExecution).toEqual({ - node3: { '0': { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] } }, - }); - expect(waitingExecutionSource).toEqual({ - node3: { - '0': { - main: [ - { previousNode: 'node1', previousNodeOutput: undefined, previousNodeRun: undefined }, - { previousNode: 'node2', previousNodeOutput: undefined, previousNodeRun: undefined }, - ], - }, - }, - }); + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); }); // ┌─────┐1 ►► @@ -287,7 +276,7 @@ describe('recreateNodeExecutionStack', () => { { from: node1, to: node3, inputIndex: 0 }, { from: node2, to: node3, inputIndex: 1 }, ); - const startNodes = [node3]; + const startNodes = new Set([node3]); const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], [node1.name]: [toITaskData([{ data: { value: 1 } }])], @@ -299,7 +288,7 @@ describe('recreateNodeExecutionStack', () => { // ACT const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); + recreateNodeExecutionStack(graph, startNodes, runData, pinData); // ASSERT expect(nodeExecutionStack).toHaveLength(1); @@ -314,22 +303,515 @@ describe('recreateNodeExecutionStack', () => { }, }); - expect(waitingExecution).toEqual({ - node3: { - '0': { - main: [[{ json: { value: 1 } }]], + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); + }); + + // ┌─────┐ ┌─────┐ + // ┌──►node1┼────┬──────► │ + // │ └─────┘ │ │merge│ + // │ │ ┌───► │ + // ├─────────────┘ │ └─────┘ + // │ │ + //┌───────┐ │ ┌─────┐ │ + //│trigger├───┴────►node2├─────┘ + //└───────┘ └─────┘ + describe('multiple inputs', () => { + // ARRANGE + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const merge = createNodeData({ name: 'merge' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, merge) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: trigger, to: merge, inputIndex: 0 }, + { from: node1, to: merge, inputIndex: 0 }, + { from: node2, to: merge, inputIndex: 1 }, + ); + + test('only the trigger has run data', () => { + // ARRANGE + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { node: 'trigger' } }])], + }; + const pinData: IPinData = {}; + const startNodes = new Set([node1, node2, merge]); + + // ACT + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(graph, startNodes, runData, pinData); + + // ASSERT + expect(nodeExecutionStack).toHaveLength(2); + expect(nodeExecutionStack[0]).toEqual({ + node: node1, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }); + expect(nodeExecutionStack[1]).toEqual({ + node: node2, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }); + + expect(waitingExecution).toEqual({ + [merge.name]: { + '0': { + main: [[{ json: { node: 'trigger' } }]], + }, }, - }, + }); + expect(waitingExecutionSource).toEqual({ + [merge.name]: { + '0': { + main: [ + { + previousNode: 'trigger', + previousNodeOutput: 0, + previousNodeRun: 0, + }, + ], + }, + }, + }); }); - expect(waitingExecutionSource).toEqual({ - node3: { - '0': { + + test('the trigger and node1 have run data', () => { + // ARRANGE + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { node: 'trigger' } }])], + [node1.name]: [toITaskData([{ data: { node: 'node1' } }])], + }; + const pinData: IPinData = {}; + const startNodes = new Set([node2, merge]); + + // ACT + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(graph, startNodes, runData, pinData); + + // ASSERT + expect(nodeExecutionStack).toHaveLength(2); + expect(nodeExecutionStack[0]).toEqual({ + node: node2, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }); + expect(nodeExecutionStack[1]).toEqual({ + node: merge, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { + main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }], + }, + }); + + expect(waitingExecution).toEqual({ + [merge.name]: { + '0': { + main: [[{ json: { node: 'node1' } }]], + }, + }, + }); + expect(waitingExecutionSource).toEqual({ + [merge.name]: { + '0': { + main: [ + { + previousNode: 'node1', + previousNodeOutput: 0, + previousNodeRun: 0, + }, + ], + }, + }, + }); + }); + + test('the trigger and node2 have run data', () => { + // ARRANGE + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { node: 'trigger' } }])], + [node2.name]: [toITaskData([{ data: { node: 'node2' } }])], + }; + const pinData: IPinData = {}; + const startNodes = new Set([node1, merge]); + + // ACT + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(graph, startNodes, runData, pinData); + + // ASSERT + expect(nodeExecutionStack).toHaveLength(2); + expect(nodeExecutionStack[0]).toEqual({ + node: node1, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }); + expect(nodeExecutionStack[1]).toEqual({ + node: merge, + data: { main: [[{ json: { node: 'trigger' } }], [{ json: { node: 'node2' } }]] }, + source: { main: [ - { previousNode: 'node1', previousNodeOutput: undefined, previousNodeRun: undefined }, - { previousNode: 'node2', previousNodeOutput: 1, previousNodeRun: undefined }, + { previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }, + { previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }, ], }, - }, + }); + + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); + }); + + test('the trigger, node1 and node2 have run data', () => { + // ARRANGE + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { node: 'trigger' } }])], + [node1.name]: [toITaskData([{ data: { node: 'node1' } }])], + [node2.name]: [toITaskData([{ data: { node: 'node2' } }])], + }; + const pinData: IPinData = {}; + const startNodes = new Set([merge]); + + // ACT + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(graph, startNodes, runData, pinData); + + // ASSERT + expect(nodeExecutionStack).toHaveLength(2); + expect(nodeExecutionStack[0]).toEqual({ + node: merge, + data: { main: [[{ json: { node: 'node1' } }], [{ json: { node: 'node2' } }]] }, + source: { + main: [ + { previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }, + { previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }, + ], + }, + }); + expect(nodeExecutionStack[1]).toEqual({ + node: merge, + data: { main: [[{ json: { node: 'trigger' } }]] }, + source: { + main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }], + }, + }); + + expect(waitingExecution).toEqual({}); + expect(waitingExecutionSource).toEqual({}); }); }); }); + +describe('addWaitingExecution', () => { + test('allow adding data partially', () => { + const waitingExecution: IWaitingForExecution = {}; + const nodeName1 = 'node 1'; + const nodeName2 = 'node 2'; + const executionData: INodeExecutionData[] = [{ json: { item: 1 } }, { json: { item: 2 } }]; + + // adding the data for the second input index first + { + addWaitingExecution( + waitingExecution, + nodeName1, + 1, // runIndex + NodeConnectionType.Main, + 1, // inputIndex + executionData, + ); + expect(waitingExecution).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [undefined, executionData], + }, + }, + }); + } + + // adding the data for the first input + { + addWaitingExecution( + waitingExecution, + nodeName1, + 1, // runIndex + NodeConnectionType.Main, + 0, // inputIndex + executionData, + ); + expect(waitingExecution).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [executionData, executionData], + }, + }, + }); + } + + // adding data for another node connection type + { + addWaitingExecution( + waitingExecution, + nodeName1, + 1, // runIndex + NodeConnectionType.AiMemory, + 0, // inputIndex + executionData, + ); + expect(waitingExecution).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [executionData, executionData], + [NodeConnectionType.AiMemory]: [executionData], + }, + }, + }); + } + + // adding data for another run + { + addWaitingExecution( + waitingExecution, + nodeName1, + 0, // runIndex + NodeConnectionType.AiChain, + 0, // inputIndex + executionData, + ); + expect(waitingExecution).toEqual({ + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [executionData], + }, + 1: { + [NodeConnectionType.Main]: [executionData, executionData], + [NodeConnectionType.AiMemory]: [executionData], + }, + }, + }); + } + + // adding data for another node + { + addWaitingExecution( + waitingExecution, + nodeName2, + 0, // runIndex + NodeConnectionType.Main, + 2, // inputIndex + executionData, + ); + expect(waitingExecution).toEqual({ + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [executionData], + }, + 1: { + [NodeConnectionType.Main]: [executionData, executionData], + [NodeConnectionType.AiMemory]: [executionData], + }, + }, + [nodeName2]: { + // runIndex + 0: { + [NodeConnectionType.Main]: [undefined, undefined, executionData], + }, + }, + }); + } + + // allow adding null + { + addWaitingExecution( + waitingExecution, + nodeName2, + 0, // runIndex + NodeConnectionType.Main, + 0, // inputIndex + null, + ); + expect(waitingExecution).toEqual({ + [nodeName2]: { + // runIndex + 0: { + [NodeConnectionType.Main]: [null, undefined, executionData], + }, + }, + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [executionData], + }, + 1: { + [NodeConnectionType.Main]: [executionData, executionData], + [NodeConnectionType.AiMemory]: [executionData], + }, + }, + }); + } + }); +}); + +describe('addWaitingExecutionSource', () => { + test('allow adding data partially', () => { + const waitingExecutionSource: IWaitingForExecutionSource = {}; + const nodeName1 = 'node 1'; + const nodeName2 = 'node 2'; + const sourceData: ISourceData = { + previousNode: 'node 0', + previousNodeRun: 0, + previousNodeOutput: 0, + }; + + // adding the data for the second input index first + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName1, + 1, // runIndex + NodeConnectionType.Main, + 1, // inputIndex + sourceData, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [undefined, sourceData], + }, + }, + }); + } + + // adding the data for the first input + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName1, + 1, // runIndex + NodeConnectionType.Main, + 0, // inputIndex + sourceData, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [sourceData, sourceData], + }, + }, + }); + } + + // adding data for another node connection type + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName1, + 1, // runIndex + NodeConnectionType.AiMemory, + 0, // inputIndex + sourceData, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 1: { + [NodeConnectionType.Main]: [sourceData, sourceData], + [NodeConnectionType.AiMemory]: [sourceData], + }, + }, + }); + } + + // adding data for another run + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName1, + 0, // runIndex + NodeConnectionType.AiChain, + 0, // inputIndex + sourceData, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [sourceData], + }, + 1: { + [NodeConnectionType.Main]: [sourceData, sourceData], + [NodeConnectionType.AiMemory]: [sourceData], + }, + }, + }); + } + + // adding data for another node + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName2, + 0, // runIndex + NodeConnectionType.Main, + 2, // inputIndex + sourceData, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [sourceData], + }, + 1: { + [NodeConnectionType.Main]: [sourceData, sourceData], + [NodeConnectionType.AiMemory]: [sourceData], + }, + }, + [nodeName2]: { + // runIndex + 0: { + [NodeConnectionType.Main]: [undefined, undefined, sourceData], + }, + }, + }); + } + + // allow adding null + { + addWaitingExecutionSource( + waitingExecutionSource, + nodeName2, + 0, // runIndex + NodeConnectionType.Main, + 0, // inputIndex + null, + ); + expect(waitingExecutionSource).toEqual({ + [nodeName1]: { + // runIndex + 0: { + [NodeConnectionType.AiChain]: [sourceData], + }, + 1: { + [NodeConnectionType.Main]: [sourceData, sourceData], + [NodeConnectionType.AiMemory]: [sourceData], + }, + }, + [nodeName2]: { + // runIndex + 0: { + [NodeConnectionType.Main]: [null, undefined, sourceData], + }, + }, + }); + } + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/cleanRunData.ts b/packages/core/src/PartialExecutionUtils/cleanRunData.ts index 5d74a3575a..bcd60c423b 100644 --- a/packages/core/src/PartialExecutionUtils/cleanRunData.ts +++ b/packages/core/src/PartialExecutionUtils/cleanRunData.ts @@ -10,7 +10,7 @@ import type { DirectedGraph } from './DirectedGraph'; export function cleanRunData( runData: IRunData, graph: DirectedGraph, - startNodes: INode[], + startNodes: Set, ): IRunData { const newRunData: IRunData = { ...runData }; diff --git a/packages/core/src/PartialExecutionUtils/findCycles.ts b/packages/core/src/PartialExecutionUtils/findCycles.ts deleted file mode 100644 index 388518ae52..0000000000 --- a/packages/core/src/PartialExecutionUtils/findCycles.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { Workflow } from 'n8n-workflow'; - -export function findCycles(_workflow: Workflow) { - // TODO: implement depth first search or Tarjan's Algorithm - return []; -} diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index a6165f6564..5eb036bd88 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -137,7 +137,7 @@ export function findStartNodes(options: { destination: INode; runData?: IRunData; pinData?: IPinData; -}): INode[] { +}): Set { const graph = options.graph; const trigger = options.trigger; const destination = options.destination; @@ -156,5 +156,5 @@ export function findStartNodes(options: { new Set(), ); - return [...startNodes]; + return startNodes; } diff --git a/packages/core/src/PartialExecutionUtils/getIncomingData.ts b/packages/core/src/PartialExecutionUtils/getIncomingData.ts index 2f5f22cd35..acac8ad22d 100644 --- a/packages/core/src/PartialExecutionUtils/getIncomingData.ts +++ b/packages/core/src/PartialExecutionUtils/getIncomingData.ts @@ -20,3 +20,26 @@ export function getIncomingData( return runData[nodeName][runIndex].data[connectionType][outputIndex]; } + +function getRunIndexLength(runData: IRunData, nodeName: string) { + return runData[nodeName]?.length ?? 0; +} + +export function getIncomingDataFromAnyRun( + runData: IRunData, + nodeName: string, + connectionType: NodeConnectionType, + outputIndex: number, +): { data: INodeExecutionData[]; runIndex: number } | undefined { + const maxRunIndexes = getRunIndexLength(runData, nodeName); + + for (let runIndex = 0; runIndex < maxRunIndexes; runIndex++) { + const data = getIncomingData(runData, nodeName, runIndex, connectionType, outputIndex); + + if (data && data.length > 0) { + return { data, runIndex }; + } + } + + return undefined; +} diff --git a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts index 58f8f2f745..d9a9940816 100644 --- a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts +++ b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts @@ -13,6 +13,25 @@ function sortByInputIndexThenByName( } } +type SourceConnectionGroup = { + /** + * This is true if all connections have data. If any connection does not have + * data it false. + * + * This is interesting to decide if a node should be put on the execution + * stack of the waiting stack in the execution engine. + */ + complete: boolean; + connections: GraphConnection[]; +}; + +function newGroup(): SourceConnectionGroup { + return { + complete: true, + connections: [], + }; +} + /** * Groups incoming connections to the node. The groups contain one connection * per input, if possible, with run data or pinned data. @@ -58,55 +77,87 @@ function sortByInputIndexThenByName( * * Since `source1` has no run data and no pinned data it's skipped in favor of * `source2` for the for input. + * It will become it's own group that is marked as `complete: false` * - * So this will return 1 group: - * 1. source2 and source3 + * So this will return 2 group: + * 1. source2 and source3, `complete: true` + * 2. source1, `complete: false` */ export function getSourceDataGroups( graph: DirectedGraph, node: INode, runData: IRunData, pinnedData: IPinData, -): GraphConnection[][] { +): SourceConnectionGroup[] { const connections = graph.getConnections({ to: node }); const sortedConnectionsWithData = []; + const sortedConnectionsWithoutData = []; for (const connection of connections) { const hasData = runData[connection.from.name] || pinnedData[connection.from.name]; if (hasData) { sortedConnectionsWithData.push(connection); + } else { + sortedConnectionsWithoutData.push(connection); } } + if (sortedConnectionsWithData.length === 0 && sortedConnectionsWithoutData.length === 0) { + return []; + } + sortedConnectionsWithData.sort(sortByInputIndexThenByName); + sortedConnectionsWithoutData.sort(sortByInputIndexThenByName); - const groups: GraphConnection[][] = []; - let currentGroup: GraphConnection[] = []; - let currentInputIndex = -1; + const groups: SourceConnectionGroup[] = []; + let currentGroup = newGroup(); + let currentInputIndex = + Math.min( + ...sortedConnectionsWithData.map((c) => c.inputIndex), + ...sortedConnectionsWithoutData.map((c) => c.inputIndex), + ) - 1; + + while (sortedConnectionsWithData.length > 0 || sortedConnectionsWithoutData.length > 0) { + currentInputIndex++; - while (sortedConnectionsWithData.length > 0) { const connectionWithDataIndex = sortedConnectionsWithData.findIndex( // eslint-disable-next-line @typescript-eslint/no-loop-func - (c) => c.inputIndex > currentInputIndex, + (c) => c.inputIndex === currentInputIndex, ); - const connection: GraphConnection | undefined = - sortedConnectionsWithData[connectionWithDataIndex]; - if (connection === undefined) { - groups.push(currentGroup); - currentGroup = []; - currentInputIndex = -1; + if (connectionWithDataIndex >= 0) { + const connection = sortedConnectionsWithData[connectionWithDataIndex]; + + currentGroup.connections.push(connection); + + sortedConnectionsWithData.splice(connectionWithDataIndex, 1); continue; } - currentInputIndex = connection.inputIndex; - currentGroup.push(connection); + const connectionWithoutDataIndex = sortedConnectionsWithoutData.findIndex( + // eslint-disable-next-line @typescript-eslint/no-loop-func + (c) => c.inputIndex === currentInputIndex, + ); - if (connectionWithDataIndex >= 0) { - sortedConnectionsWithData.splice(connectionWithDataIndex, 1); + if (connectionWithoutDataIndex >= 0) { + const connection = sortedConnectionsWithoutData[connectionWithoutDataIndex]; + + currentGroup.connections.push(connection); + currentGroup.complete = false; + + sortedConnectionsWithoutData.splice(connectionWithoutDataIndex, 1); + continue; } + + groups.push(currentGroup); + currentGroup = newGroup(); + currentInputIndex = + Math.min( + ...sortedConnectionsWithData.map((c) => c.inputIndex), + ...sortedConnectionsWithoutData.map((c) => c.inputIndex), + ) - 1; } groups.push(currentGroup); diff --git a/packages/core/src/PartialExecutionUtils/handleCycles.ts b/packages/core/src/PartialExecutionUtils/handleCycles.ts new file mode 100644 index 0000000000..94a8ae8cbc --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/handleCycles.ts @@ -0,0 +1,56 @@ +import type { INode } from 'n8n-workflow'; +import * as a from 'node:assert/strict'; + +import type { DirectedGraph } from './DirectedGraph'; + +/** + * Returns a new set of start nodes. + * + * For every start node this checks if it is part of a cycle and if it is it + * replaces the start node with the start of the cycle. + * + * This is useful because it prevents executing cycles partially, e.g. figuring + * our which run of the cycle has to be repeated etc. + */ +export function handleCycles( + graph: DirectedGraph, + startNodes: Set, + trigger: INode, +): Set { + // Strongly connected components can also be nodes that are not part of a + // cycle. They form a strongly connected component of one. E.g the trigger is + // always a strongly connected component by itself because it does not have + // any inputs and thus cannot build a cycle. + // + // We're not interested in them so we filter them out. + const cycles = graph.getStronglyConnectedComponents().filter((cycle) => cycle.size >= 1); + const newStartNodes: Set = new Set(startNodes); + + // For each start node, check if the node is part of a cycle and if it is + // replace the start node with the start of the cycle. + if (cycles.length === 0) { + return newStartNodes; + } + + for (const startNode of startNodes) { + for (const cycle of cycles) { + const isPartOfCycle = cycle.has(startNode); + if (isPartOfCycle) { + const firstNode = graph.depthFirstSearch({ + from: trigger, + fn: (node) => cycle.has(node), + }); + + a.ok( + firstNode, + "the trigger must be connected to the cycle, otherwise the cycle wouldn't be part of the subgraph", + ); + + newStartNodes.delete(startNode); + newStartNodes.add(firstNode); + } + } + } + + return newStartNodes; +} diff --git a/packages/core/src/PartialExecutionUtils/index.ts b/packages/core/src/PartialExecutionUtils/index.ts index 6a6f1a233a..cea8ded9b9 100644 --- a/packages/core/src/PartialExecutionUtils/index.ts +++ b/packages/core/src/PartialExecutionUtils/index.ts @@ -2,5 +2,6 @@ export { DirectedGraph } from './DirectedGraph'; export { findTriggerForPartialExecution } from './findTriggerForPartialExecution'; export { findStartNodes } from './findStartNodes'; export { findSubgraph } from './findSubgraph'; -export { findCycles } from './findCycles'; export { recreateNodeExecutionStack } from './recreateNodeExecutionStack'; +export { cleanRunData } from './cleanRunData'; +export { handleCycles } from './handleCycles'; diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index 4926becb79..542d4b8fbd 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -13,9 +13,47 @@ import { } from 'n8n-workflow'; import type { DirectedGraph } from './DirectedGraph'; -import { getIncomingData } from './getIncomingData'; +import { getIncomingDataFromAnyRun } from './getIncomingData'; import { getSourceDataGroups } from './getSourceDataGroups'; +export function addWaitingExecution( + waitingExecution: IWaitingForExecution, + nodeName: string, + runIndex: number, + inputType: NodeConnectionType, + inputIndex: number, + executionData: INodeExecutionData[] | null, +) { + const waitingExecutionObject = waitingExecution[nodeName] ?? {}; + const taskDataConnections = waitingExecutionObject[runIndex] ?? {}; + const executionDataList = taskDataConnections[inputType] ?? []; + + executionDataList[inputIndex] = executionData; + + taskDataConnections[inputType] = executionDataList; + waitingExecutionObject[runIndex] = taskDataConnections; + waitingExecution[nodeName] = waitingExecutionObject; +} + +export function addWaitingExecutionSource( + waitingExecutionSource: IWaitingForExecutionSource, + nodeName: string, + runIndex: number, + inputType: NodeConnectionType, + inputIndex: number, + sourceData: ISourceData | null, +) { + const waitingExecutionSourceObject = waitingExecutionSource[nodeName] ?? {}; + const taskDataConnectionsSource = waitingExecutionSourceObject[runIndex] ?? {}; + const sourceDataList = taskDataConnectionsSource[inputType] ?? []; + + sourceDataList[inputIndex] = sourceData; + + taskDataConnectionsSource[inputType] = sourceDataList; + waitingExecutionSourceObject[runIndex] = taskDataConnectionsSource; + waitingExecutionSource[nodeName] = waitingExecutionSourceObject; +} + /** * Recreates the node execution stack, waiting executions and waiting * execution sources from a directed graph, start nodes, the destination node, @@ -32,8 +70,7 @@ import { getSourceDataGroups } from './getSourceDataGroups'; */ export function recreateNodeExecutionStack( graph: DirectedGraph, - startNodes: INode[], - destinationNode: INode, + startNodes: Set, runData: IRunData, pinData: IPinData, ): { @@ -59,9 +96,6 @@ export function recreateNodeExecutionStack( const waitingExecution: IWaitingForExecution = {}; const waitingExecutionSource: IWaitingForExecutionSource = {}; - // TODO: Don't hard code this! - const runIndex = 0; - for (const startNode of startNodes) { const incomingStartNodeConnections = graph .getDirectParentConnections(startNode) @@ -84,89 +118,94 @@ export function recreateNodeExecutionStack( const sourceDataSets = getSourceDataGroups(graph, startNode, runData, pinData); for (const sourceData of sourceDataSets) { - incomingData = []; + if (sourceData.complete) { + // All incoming connections have data, so let's put the node on the + // stack! + incomingData = []; - incomingSourceData = { main: [] }; + incomingSourceData = { main: [] }; - for (const incomingConnection of sourceData) { - const node = incomingConnection.from; + for (const incomingConnection of sourceData.connections) { + let runIndex = 0; + const sourceNode = incomingConnection.from; - if (pinData[node.name]) { - incomingData.push(pinData[node.name]); - } else { - a.ok( - runData[node.name], - `Start node(${incomingConnection.to.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.name}". Are you sure the start nodes come from the "findStartNodes" function?`, - ); + if (pinData[sourceNode.name]) { + incomingData.push(pinData[sourceNode.name]); + } else { + a.ok( + runData[sourceNode.name], + `Start node(${incomingConnection.to.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${sourceNode.name}->${startNode.name}". Are you sure the start nodes come from the "findStartNodes" function?`, + ); - const nodeIncomingData = getIncomingData( + const nodeIncomingData = getIncomingDataFromAnyRun( + runData, + sourceNode.name, + incomingConnection.type, + incomingConnection.outputIndex, + ); + + if (nodeIncomingData) { + runIndex = nodeIncomingData.runIndex; + incomingData.push(nodeIncomingData.data); + } + } + + incomingSourceData.main.push({ + previousNode: incomingConnection.from.name, + previousNodeOutput: incomingConnection.outputIndex, + previousNodeRun: runIndex, + }); + } + + const executeData: IExecuteData = { + node: startNode, + data: { main: incomingData }, + source: incomingSourceData, + }; + + nodeExecutionStack.push(executeData); + } else { + const nodeName = startNode.name; + const nextRunIndex = waitingExecution[nodeName] + ? Object.keys(waitingExecution[nodeName]).length + : 0; + + for (const incomingConnection of sourceData.connections) { + const sourceNode = incomingConnection.from; + const maybeNodeIncomingData = getIncomingDataFromAnyRun( runData, - node.name, - runIndex, + sourceNode.name, incomingConnection.type, incomingConnection.outputIndex, ); + const nodeIncomingData = maybeNodeIncomingData?.data ?? null; if (nodeIncomingData) { - incomingData.push(nodeIncomingData); + addWaitingExecution( + waitingExecution, + nodeName, + nextRunIndex, + incomingConnection.type, + incomingConnection.inputIndex, + nodeIncomingData, + ); + + addWaitingExecutionSource( + waitingExecutionSource, + nodeName, + nextRunIndex, + incomingConnection.type, + incomingConnection.inputIndex, + nodeIncomingData + ? { + previousNode: incomingConnection.from.name, + previousNodeRun: nextRunIndex, + previousNodeOutput: incomingConnection.outputIndex, + } + : null, + ); } } - - incomingSourceData.main.push({ - previousNode: incomingConnection.from.name, - previousNodeOutput: incomingConnection.outputIndex, - previousNodeRun: 0, - }); - } - - const executeData: IExecuteData = { - node: startNode, - data: { main: incomingData }, - source: incomingSourceData, - }; - - nodeExecutionStack.push(executeData); - } - } - - // TODO: Do we need this? - if (destinationNode) { - const destinationNodeName = destinationNode.name; - // Check if the destinationNode has to be added as waiting - // because some input data is already fully available - const incomingDestinationNodeConnections = graph - .getDirectParentConnections(destinationNode) - .filter((c) => c.type === NodeConnectionType.Main); - if (incomingDestinationNodeConnections !== undefined) { - for (const connection of incomingDestinationNodeConnections) { - if (waitingExecution[destinationNodeName] === undefined) { - waitingExecution[destinationNodeName] = {}; - waitingExecutionSource[destinationNodeName] = {}; - } - if (waitingExecution[destinationNodeName][runIndex] === undefined) { - waitingExecution[destinationNodeName][runIndex] = {}; - waitingExecutionSource[destinationNodeName][runIndex] = {}; - } - if (waitingExecution[destinationNodeName][runIndex][connection.type] === undefined) { - waitingExecution[destinationNodeName][runIndex][connection.type] = []; - waitingExecutionSource[destinationNodeName][runIndex][connection.type] = []; - } - - if (runData[connection.from.name] !== undefined) { - // Input data exists so add as waiting - // incomingDataDestination.push(runData[connection.node!][runIndex].data![connection.type][connection.index]); - waitingExecution[destinationNodeName][runIndex][connection.type].push( - runData[connection.from.name][runIndex].data![connection.type][connection.inputIndex], - ); - waitingExecutionSource[destinationNodeName][runIndex][connection.type].push({ - previousNode: connection.from.name, - previousNodeOutput: connection.inputIndex || undefined, - previousNodeRun: runIndex || undefined, - } as ISourceData); - } else { - waitingExecution[destinationNodeName][runIndex][connection.type].push(null); - waitingExecutionSource[destinationNodeName][runIndex][connection.type].push(null); - } } } } diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index ec5963a54b..23b88abd5b 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -51,13 +51,13 @@ import PCancelable from 'p-cancelable'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import { DirectedGraph, - findCycles, findStartNodes, findSubgraph, findTriggerForPartialExecution, + cleanRunData, + recreateNodeExecutionStack, + handleCycles, } from './PartialExecutionUtils'; -import { cleanRunData } from './PartialExecutionUtils/cleanRunData'; -import { recreateNodeExecutionStack } from './PartialExecutionUtils/recreateNodeExecutionStack'; export class WorkflowExecute { private status: ExecutionStatus = 'new'; @@ -352,22 +352,18 @@ export class WorkflowExecute { const filteredNodes = subgraph.getNodes(); // 3. Find the Start Nodes - const startNodes = findStartNodes({ graph: subgraph, trigger, destination, runData }); + let startNodes = findStartNodes({ graph: subgraph, trigger, destination, runData }); // 4. Detect Cycles - const cycles = findCycles(workflow); - // 5. Handle Cycles - if (cycles.length) { - // TODO: handle - } + startNodes = handleCycles(graph, startNodes, trigger); // 6. Clean Run Data const newRunData: IRunData = cleanRunData(runData, graph, startNodes); // 7. Recreate Execution Stack const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(subgraph, startNodes, destination, runData, pinData ?? {}); + recreateNodeExecutionStack(subgraph, new Set(startNodes), runData, pinData ?? {}); // 8. Execute this.status = 'running'; diff --git a/packages/core/test/InstanceSettings.test.ts b/packages/core/test/InstanceSettings.test.ts index 64b6840f2f..22f8fadff8 100644 --- a/packages/core/test/InstanceSettings.test.ts +++ b/packages/core/test/InstanceSettings.test.ts @@ -1,20 +1,35 @@ import fs from 'fs'; -import { InstanceSettings } from '@/InstanceSettings'; +import { InstanceSettings } from '../src/InstanceSettings'; +import { InstanceSettingsConfig } from '../src/InstanceSettingsConfig'; describe('InstanceSettings', () => { process.env.N8N_USER_FOLDER = '/test'; const existSpy = jest.spyOn(fs, 'existsSync'); - beforeEach(() => jest.resetAllMocks()); + const statSpy = jest.spyOn(fs, 'statSync'); + const chmodSpy = jest.spyOn(fs, 'chmodSync'); + + const createSettingsInstance = (opts?: Partial) => + new InstanceSettings({ + ...new InstanceSettingsConfig(), + ...opts, + }); + + beforeEach(() => { + jest.resetAllMocks(); + statSpy.mockReturnValue({ mode: 0o600 } as fs.Stats); + }); describe('If the settings file exists', () => { const readSpy = jest.spyOn(fs, 'readFileSync'); - beforeEach(() => existSpy.mockReturnValue(true)); + beforeEach(() => { + existSpy.mockReturnValue(true); + }); it('should load settings from the file', () => { readSpy.mockReturnValue(JSON.stringify({ encryptionKey: 'test_key' })); - const settings = new InstanceSettings(); + const settings = createSettingsInstance(); expect(settings.encryptionKey).toEqual('test_key'); expect(settings.instanceId).toEqual( '6ce26c63596f0cc4323563c529acfca0cccb0e57f6533d79a60a42c9ff862ae7', @@ -23,13 +38,52 @@ describe('InstanceSettings', () => { it('should throw error if settings file is not valid JSON', () => { readSpy.mockReturnValue('{"encryptionKey":"test_key"'); - expect(() => new InstanceSettings()).toThrowError(); + expect(() => createSettingsInstance()).toThrowError(); }); it('should throw if the env and file keys do not match', () => { readSpy.mockReturnValue(JSON.stringify({ encryptionKey: 'key_1' })); process.env.N8N_ENCRYPTION_KEY = 'key_2'; - expect(() => new InstanceSettings()).toThrowError(); + expect(() => createSettingsInstance()).toThrowError(); + }); + + it('should check if the settings file has the correct permissions', () => { + process.env.N8N_ENCRYPTION_KEY = 'test_key'; + readSpy.mockReturnValueOnce(JSON.stringify({ encryptionKey: 'test_key' })); + statSpy.mockReturnValueOnce({ mode: 0o600 } as fs.Stats); + const settings = createSettingsInstance(); + expect(settings.encryptionKey).toEqual('test_key'); + expect(settings.instanceId).toEqual( + '6ce26c63596f0cc4323563c529acfca0cccb0e57f6533d79a60a42c9ff862ae7', + ); + expect(statSpy).toHaveBeenCalledWith('/test/.n8n/config'); + }); + + it('should check the permissions but not fix them if settings file has incorrect permissions by default', () => { + readSpy.mockReturnValueOnce(JSON.stringify({ encryptionKey: 'test_key' })); + statSpy.mockReturnValueOnce({ mode: 0o644 } as fs.Stats); + createSettingsInstance(); + expect(statSpy).toHaveBeenCalledWith('/test/.n8n/config'); + expect(chmodSpy).not.toHaveBeenCalled(); + }); + + it("should not check the permissions if 'N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS' is false", () => { + process.env.N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS = 'false'; + readSpy.mockReturnValueOnce(JSON.stringify({ encryptionKey: 'test_key' })); + createSettingsInstance(); + expect(statSpy).not.toHaveBeenCalled(); + expect(chmodSpy).not.toHaveBeenCalled(); + }); + + it("should fix the permissions of the settings file if 'N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS' is true", () => { + process.env.N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS = 'true'; + readSpy.mockReturnValueOnce(JSON.stringify({ encryptionKey: 'test_key' })); + statSpy.mockReturnValueOnce({ mode: 0o644 } as fs.Stats); + createSettingsInstance({ + enforceSettingsFilePermissions: true, + }); + expect(statSpy).toHaveBeenCalledWith('/test/.n8n/config'); + expect(chmodSpy).toHaveBeenCalledWith('/test/.n8n/config', 0o600); }); }); @@ -42,20 +96,58 @@ describe('InstanceSettings', () => { writeFileSpy.mockReturnValue(); }); - it('should create a new settings file', () => { - const settings = new InstanceSettings(); + it('should create a new settings file without explicit permissions if N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS is not set', () => { + process.env.N8N_ENCRYPTION_KEY = 'key_2'; + const settings = createSettingsInstance(); expect(settings.encryptionKey).not.toEqual('test_key'); expect(mkdirSpy).toHaveBeenCalledWith('/test/.n8n', { recursive: true }); expect(writeFileSpy).toHaveBeenCalledWith( '/test/.n8n/config', expect.stringContaining('"encryptionKey":'), - 'utf-8', + { + encoding: 'utf-8', + mode: undefined, + }, + ); + }); + + it('should create a new settings file without explicit permissions if N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=false', () => { + process.env.N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS = 'false'; + process.env.N8N_ENCRYPTION_KEY = 'key_2'; + const settings = createSettingsInstance(); + expect(settings.encryptionKey).not.toEqual('test_key'); + expect(mkdirSpy).toHaveBeenCalledWith('/test/.n8n', { recursive: true }); + expect(writeFileSpy).toHaveBeenCalledWith( + '/test/.n8n/config', + expect.stringContaining('"encryptionKey":'), + { + encoding: 'utf-8', + mode: undefined, + }, + ); + }); + + it('should create a new settings file with explicit permissions if N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true', () => { + process.env.N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS = 'true'; + process.env.N8N_ENCRYPTION_KEY = 'key_2'; + const settings = createSettingsInstance({ + enforceSettingsFilePermissions: true, + }); + expect(settings.encryptionKey).not.toEqual('test_key'); + expect(mkdirSpy).toHaveBeenCalledWith('/test/.n8n', { recursive: true }); + expect(writeFileSpy).toHaveBeenCalledWith( + '/test/.n8n/config', + expect.stringContaining('"encryptionKey":'), + { + encoding: 'utf-8', + mode: 0o600, + }, ); }); it('should pick up the encryption key from env var N8N_ENCRYPTION_KEY', () => { process.env.N8N_ENCRYPTION_KEY = 'env_key'; - const settings = new InstanceSettings(); + const settings = createSettingsInstance(); expect(settings.encryptionKey).toEqual('env_key'); expect(settings.instanceId).toEqual( '2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683', @@ -65,8 +157,42 @@ describe('InstanceSettings', () => { expect(writeFileSpy).toHaveBeenCalledWith( '/test/.n8n/config', expect.stringContaining('"encryptionKey":'), - 'utf-8', + { + encoding: 'utf-8', + mode: undefined, + }, + ); + }); + + it("should not set the permissions of the settings file if 'N8N_IGNORE_SETTINGS_FILE_PERMISSIONS' is true", () => { + process.env.N8N_ENCRYPTION_KEY = 'key_2'; + process.env.N8N_IGNORE_SETTINGS_FILE_PERMISSIONS = 'true'; + const settings = createSettingsInstance(); + expect(settings.encryptionKey).not.toEqual('test_key'); + expect(mkdirSpy).toHaveBeenCalledWith('/test/.n8n', { recursive: true }); + expect(writeFileSpy).toHaveBeenCalledWith( + '/test/.n8n/config', + expect.stringContaining('"encryptionKey":'), + { + encoding: 'utf-8', + mode: undefined, + }, ); }); }); + + describe('constructor', () => { + it('should generate a `hostId`', () => { + const encryptionKey = 'test_key'; + process.env.N8N_ENCRYPTION_KEY = encryptionKey; + jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true); + jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(JSON.stringify({ encryptionKey })); + + const settings = createSettingsInstance(); + + const [instanceType, nanoid] = settings.hostId.split('-'); + expect(instanceType).toEqual('main'); + expect(nanoid).toHaveLength(16); // e.g. sDX6ZPc0bozv66zM + }); + }); }); diff --git a/packages/core/test/utils.ts b/packages/core/test/utils.ts index 7f4862cabd..f1ed54dd03 100644 --- a/packages/core/test/utils.ts +++ b/packages/core/test/utils.ts @@ -1,12 +1,11 @@ import { mock } from 'jest-mock-extended'; import { Duplex } from 'stream'; import type { DeepPartial } from 'ts-essentials'; +import type { Constructable } from 'typedi'; import { Container } from 'typedi'; -import type { Class } from '@/Interfaces'; - export const mockInstance = ( - constructor: Class, + constructor: Constructable, data: DeepPartial | undefined = undefined, ) => { const instance = mock(data); diff --git a/packages/design-system/package.json b/packages/design-system/package.json index 36fcf31528..8aec0ab60f 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -1,6 +1,6 @@ { "name": "n8n-design-system", - "version": "1.53.0", + "version": "1.55.0", "main": "src/main.ts", "import": "src/main.ts", "scripts": { diff --git a/packages/design-system/src/__tests__/setup.ts b/packages/design-system/src/__tests__/setup.ts index 6eb1c426fc..981c9d5a60 100644 --- a/packages/design-system/src/__tests__/setup.ts +++ b/packages/design-system/src/__tests__/setup.ts @@ -1,8 +1,11 @@ import '@testing-library/jest-dom'; +import { configure } from '@testing-library/vue'; import { config } from '@vue/test-utils'; import { N8nPlugin } from 'n8n-design-system/plugin'; +configure({ testIdAttribute: 'data-test-id' }); + config.global.plugins = [N8nPlugin]; window.ResizeObserver = diff --git a/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue index 1acffa82fa..78b4ccbf51 100644 --- a/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue +++ b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue @@ -226,6 +226,16 @@ async function onCopyButtonClick(content: string, e: MouseEvent) { data-test-id="chat-message-system" > ⚠️ {{ message.content }} + + {{ t('generic.retry') }} +
{ }); expect(container).toMatchSnapshot(); }); + + it('renders error message correctly with retry button', () => { + const wrapper = render(AskAssistantChat, { + global: { + directives: { + n8nHtml, + }, + stubs, + }, + props: { + user: { firstName: 'Kobi', lastName: 'Dog' }, + messages: [ + { + id: '1', + role: 'assistant', + type: 'error', + content: 'This is an error message.', + read: false, + // Button is not shown without a retry function + retry: async () => {}, + }, + ], + }, + }); + expect(wrapper.container).toMatchSnapshot(); + expect(wrapper.getByTestId('error-retry-button')).toBeInTheDocument(); + }); + + it('does not render retry button if no error is present', () => { + const wrapper = render(AskAssistantChat, { + global: { + directives: { + n8nHtml, + }, + stubs, + }, + props: { + user: { firstName: 'Kobi', lastName: 'Dog' }, + messages: [ + { + id: '1', + type: 'text', + role: 'assistant', + content: + 'Hi Max! Here is my top solution to fix the error in your **Transform data** node👇', + read: false, + }, + ], + }, + }); + + expect(wrapper.container).toMatchSnapshot(); + expect(wrapper.queryByTestId('error-retry-button')).not.toBeInTheDocument(); + }); }); diff --git a/packages/design-system/src/components/AskAssistantChat/__tests__/__snapshots__/AskAssistantChat.spec.ts.snap b/packages/design-system/src/components/AskAssistantChat/__tests__/__snapshots__/AskAssistantChat.spec.ts.snap index 891c10abf6..c26913e405 100644 --- a/packages/design-system/src/components/AskAssistantChat/__tests__/__snapshots__/AskAssistantChat.spec.ts.snap +++ b/packages/design-system/src/components/AskAssistantChat/__tests__/__snapshots__/AskAssistantChat.spec.ts.snap @@ -1,5 +1,179 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`AskAssistantChat > does not render retry button if no error is present 1`] = ` +
+
+
+
+
+ + + + + + + + + + + + AI Assistant + +
+
+ beta +
+
+
+ +
+
+
+
+ +
+
+
+ + + + + + + + + + +
+ + Assistant + +
+
+
+

+ Hi Max! Here is my top solution to fix the error in your + + Transform data + + node👇 +

+ + +
+ + +
+ +
+ +
+ +
+
+