diff --git a/.github/scripts/package.json b/.github/scripts/package.json index cd67711fa4..80bf0baf11 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -5,7 +5,7 @@ "debug": "4.3.4", "glob": "10.3.10", "p-limit": "3.1.0", - "picocolors": "1.0.0", + "picocolors": "1.0.1", "semver": "7.5.4", "tempfile": "5.0.0", "typescript": "*" diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml index add6e858a3..761cc5b8c7 100644 --- a/.github/workflows/check-pr-title.yml +++ b/.github/workflows/check-pr-title.yml @@ -7,8 +7,7 @@ on: - edited - synchronize branches: - - '**' - - '!release/*' + - 'master' jobs: check-pr-title: diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.yml index bfd43e9257..d2ebc05320 100644 --- a/.github/workflows/chromatic.yml +++ b/.github/workflows/chromatic.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: pull_request_review: types: [submitted] - branch: + branches: - 'master' paths: - packages/design-system/** diff --git a/.github/workflows/ci-postgres-mysql.yml b/.github/workflows/ci-postgres-mysql.yml index d5abf1fa26..1d645ae774 100644 --- a/.github/workflows/ci-postgres-mysql.yml +++ b/.github/workflows/ci-postgres-mysql.yml @@ -8,6 +8,10 @@ on: paths: - packages/cli/src/databases/** - .github/workflows/ci-postgres-mysql.yml + pull_request_review: + types: [submitted] + branches: + - 'release/*' concurrency: group: db-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index 78a5388950..d9938983b1 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -1,6 +1,10 @@ name: Build, unit test and lint branch -on: [pull_request] +on: + pull_request: + branches: + - '**' + - '!release/*' jobs: install-and-build: @@ -9,7 +13,6 @@ jobs: steps: - uses: actions/checkout@v4.1.1 with: - repository: n8n-io/n8n ref: refs/pull/${{ github.event.pull_request.number }}/merge - run: corepack enable diff --git a/.github/workflows/docker-images-benchmark.yml b/.github/workflows/docker-images-benchmark.yml new file mode 100644 index 0000000000..cf9d7359a5 --- /dev/null +++ b/.github/workflows/docker-images-benchmark.yml @@ -0,0 +1,43 @@ +name: Benchmark Docker Image CI + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'packages/@n8n/benchmark/**' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - '.github/workflows/docker-images-benchmark.yml' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4.1.1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build + uses: docker/build-push-action@v5.1.0 + with: + context: . + file: ./packages/@n8n/benchmark/Dockerfile + platforms: linux/amd64 + provenance: false + push: true + tags: | + ghcr.io/${{ github.repository_owner }}/n8n-benchmark:latest diff --git a/.github/workflows/docker-images-nightly.yml b/.github/workflows/docker-images-nightly.yml index b701392e8b..34950d7ac2 100644 --- a/.github/workflows/docker-images-nightly.yml +++ b/.github/workflows/docker-images-nightly.yml @@ -6,10 +6,6 @@ on: - cron: '0 1 * * *' workflow_dispatch: inputs: - repository: - description: 'GitHub repository to create image off.' - required: true - default: 'n8n-io/n8n' branch: description: 'GitHub branch to create image off.' required: true @@ -49,7 +45,6 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.1 with: - repository: ${{ github.event.inputs.repository || 'n8n-io/n8n' }} ref: ${{ github.event.inputs.branch || 'master' }} - name: Set up QEMU diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml deleted file mode 100644 index 5c91f3832d..0000000000 --- a/.github/workflows/docker-images.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Docker Image CI - -on: - release: - types: [published] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4.1.1 - - - name: Get the version - id: vars - run: echo ::set-output name=tag::$(echo ${GITHUB_REF:14}) - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3.0.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to DockerHub - uses: docker/login-action@v3.0.0 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build - uses: docker/build-push-action@v5.1.0 - with: - context: ./docker/images/n8n - build-args: | - N8N_VERSION=${{ steps.vars.outputs.tag }} - platforms: linux/amd64,linux/arm64 - provenance: false - push: true - tags: | - ${{ secrets.DOCKER_USERNAME }}/n8n:${{ steps.vars.outputs.tag }} - ghcr.io/${{ github.repository_owner }}/n8n:${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml index beea3bca27..ab88930a3a 100644 --- a/.github/workflows/e2e-reusable.yml +++ b/.github/workflows/e2e-reusable.yml @@ -22,11 +22,6 @@ on: required: false default: 'browsers:node18.12.0-chrome107' type: string - cache-key: - description: 'Cache key for modules and build artifacts.' - required: false - default: ${{ github.sha }}-${{ inputs.run-env }}-e2e-modules - type: string record: description: 'Record test run.' required: false @@ -78,7 +73,6 @@ jobs: steps: - uses: actions/checkout@v4.1.1 with: - repository: n8n-io/n8n ref: ${{ inputs.branch }} - name: Checkout PR @@ -111,7 +105,7 @@ jobs: /github/home/.cache /github/home/.pnpm-store ./packages/**/dist - key: ${{ inputs.cache-key }} + key: ${{ github.sha }}-e2e testing: runs-on: ubuntu-latest @@ -128,7 +122,6 @@ jobs: steps: - uses: actions/checkout@v4.1.1 with: - repository: n8n-io/n8n ref: ${{ inputs.branch }} - name: Checkout PR @@ -146,7 +139,7 @@ jobs: /github/home/.cache /github/home/.pnpm-store ./packages/**/dist - key: ${{ inputs.cache-key }} + key: ${{ github.sha }}-e2e - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/e2e-tests-pr.yml b/.github/workflows/e2e-tests-pr.yml index f845dbb062..3d2f122638 100644 --- a/.github/workflows/e2e-tests-pr.yml +++ b/.github/workflows/e2e-tests-pr.yml @@ -3,8 +3,9 @@ name: PR E2E on: pull_request_review: types: [submitted] - branch: + branches: - 'master' + - 'release/*' concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} @@ -18,7 +19,6 @@ jobs: with: pr_number: ${{ github.event.pull_request.number }} user: ${{ github.event.pull_request.user.login || 'PR User' }} - spec: 'e2e/*' secrets: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} diff --git a/.github/workflows/linting-reusable.yml b/.github/workflows/linting-reusable.yml index 2650622bd0..ed8d234940 100644 --- a/.github/workflows/linting-reusable.yml +++ b/.github/workflows/linting-reusable.yml @@ -21,7 +21,6 @@ jobs: steps: - uses: actions/checkout@v4.1.1 with: - repository: n8n-io/n8n ref: ${{ inputs.ref }} - run: corepack enable diff --git a/.github/workflows/release-create-pr.yml b/.github/workflows/release-create-pr.yml index a17fa1bf89..03572e541c 100644 --- a/.github/workflows/release-create-pr.yml +++ b/.github/workflows/release-create-pr.yml @@ -56,12 +56,12 @@ jobs: git push -f origin refs/remotes/origin/${{ github.event.inputs.base-branch }}:refs/heads/release/${{ env.NEXT_RELEASE }} - name: Push the release branch, and Create the PR - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: base: 'release/${{ env.NEXT_RELEASE }}' - branch: '${{ env.NEXT_RELEASE }}-pr' + branch: 'release-pr/${{ env.NEXT_RELEASE }}' commit-message: ':rocket: Release ${{ env.NEXT_RELEASE }}' delete-branch: true - labels: 'release' + labels: release,release:${{ github.event.inputs.release-type }} title: ':rocket: Release ${{ env.NEXT_RELEASE }}' body-path: 'CHANGELOG-${{ env.NEXT_RELEASE }}.md' diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 239c18b512..522d47ff0f 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -8,18 +8,15 @@ on: - 'release/*' jobs: - publish-release: - if: github.event.pull_request.merged == true + publish-to-npm: + name: Publish to NPM runs-on: ubuntu-latest - - permissions: - contents: write - id-token: write - - timeout-minutes: 60 + if: github.event.pull_request.merged == true + timeout-minutes: 10 env: NPM_CONFIG_PROVENANCE: true - + outputs: + release: ${{ steps.set-release.outputs.release }} steps: - name: Checkout uses: actions/checkout@v4.1.1 @@ -51,25 +48,97 @@ jobs: pnpm publish -r --publish-branch ${{github.event.pull_request.base.ref}} --access public --tag rc --no-git-checks npm dist-tag rm n8n rc + - id: set-release + run: echo "release=${{ env.RELEASE }}" >> $GITHUB_OUTPUT + + publish-to-docker-hub: + name: Publish to DockerHub + needs: [publish-to-npm] + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@v3.0.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build + uses: docker/build-push-action@v5.1.0 + with: + context: ./docker/images/n8n + build-args: | + N8N_VERSION=${{ needs.publish-to-npm.outputs.release }} + platforms: linux/amd64,linux/arm64 + provenance: false + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/n8n:${{ needs.publish-to-npm.outputs.release }} + ghcr.io/${{ github.repository_owner }}/n8n:${{ needs.publish-to-npm.outputs.release }} + + create-github-release: + name: Create a GitHub Release + needs: [publish-to-npm, publish-to-docker-hub] + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + timeout-minutes: 5 + + permissions: + contents: write + id-token: write + + steps: - name: Create a Release on GitHub uses: ncipollo/release-action@v1 with: commit: ${{github.event.pull_request.base.ref}} - tag: 'n8n@${{env.RELEASE}}' + tag: 'n8n@${{ needs.publish-to-npm.outputs.release }}' prerelease: true makeLatest: false body: ${{github.event.pull_request.body}} + trigger-release-note: + name: Trigger a release note + needs: [publish-to-npm, create-github-release] + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: - name: Trigger a release note - continue-on-error: true - run: curl -u docsWorkflows:${{ secrets.N8N_WEBHOOK_DOCS_PASSWORD }} --request GET 'https://internal.users.n8n.cloud/webhook/trigger-release-note' --header 'Content-Type:application/json' --data '{"version":"${{env.RELEASE}}"}' + run: curl -u docsWorkflows:${{ secrets.N8N_WEBHOOK_DOCS_PASSWORD }} --request GET 'https://internal.users.n8n.cloud/webhook/trigger-release-note' --header 'Content-Type:application/json' --data '{"version":"${{ needs.publish-to-npm.outputs.release }}"}' - # - name: Merge Release into 'master' - # run: | - # git fetch origin - # git checkout --track origin/master - # git config user.name "Jan Oberhauser" - # git config user.email jan.oberhauser@gmail.com - # git merge --ff n8n@${{env.RELEASE}} - # git push origin master - # git push origin :${{github.event.pull_request.base.ref}} + merge-back-into-master: + name: Merge back into master + needs: [publish-to-npm, create-github-release] + if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'release:patch') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - run: | + git checkout --track origin/master + git config user.name "github-actions[bot]" + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git merge --ff n8n@${{ needs.publish-to-npm.outputs.release }} + git push origin master + git push origin :${{github.event.pull_request.base.ref}} diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 8ba22b590c..2bb91dd065 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -73,6 +73,7 @@ jobs: env: N8N_ENCRYPTION_KEY: ${{secrets.ENCRYPTION_KEY}} SKIP_STATISTICS_EVENTS: true + DB_SQLITE_POOL_SIZE: 4 # - # name: Export credentials # if: always() diff --git a/.github/workflows/units-tests-reusable.yml b/.github/workflows/units-tests-reusable.yml index fe270b1de0..60bf593e82 100644 --- a/.github/workflows/units-tests-reusable.yml +++ b/.github/workflows/units-tests-reusable.yml @@ -36,7 +36,6 @@ jobs: steps: - uses: actions/checkout@v4.1.1 with: - repository: n8n-io/n8n ref: ${{ inputs.ref }} - run: corepack enable @@ -50,6 +49,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Setup build cache + if: inputs.collectCoverage != true uses: rharkor/caching-for-turbo@v1.5 - name: Build diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2a07dda0..08aadc87da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,169 @@ +# [1.56.0](https://github.com/n8n-io/n8n/compare/n8n@1.55.0...n8n@1.56.0) (2024-08-21) + + +### Bug Fixes + +* Better errors in Switch, If and Filter nodes ([#10457](https://github.com/n8n-io/n8n/issues/10457)) ([aea82cb](https://github.com/n8n-io/n8n/commit/aea82cb74421d516919742127daf669808b57604)) +* **Calendly Trigger Node:** Fix issue with webhook url matching ([#10378](https://github.com/n8n-io/n8n/issues/10378)) ([09c3a8b](https://github.com/n8n-io/n8n/commit/09c3a8b36733a9634ef5948922d6aa7a19bbb592)) +* **core:** Fix payload property in `workflow-post-execute` event ([#10413](https://github.com/n8n-io/n8n/issues/10413)) ([d98e29e](https://github.com/n8n-io/n8n/commit/d98e29e3d53de87aec276260615fa60473a2692f)) +* **core:** Fix XSS validation and separate URL validation ([#10424](https://github.com/n8n-io/n8n/issues/10424)) ([91467ab](https://github.com/n8n-io/n8n/commit/91467ab325e4c71c20c522f3143246d270101626)) +* **core:** Replace `sanitize-html` with `xss` in XSS validator constraint ([#10479](https://github.com/n8n-io/n8n/issues/10479)) ([5dea51a](https://github.com/n8n-io/n8n/commit/5dea51aad7d9e7ffc676d16f4bbbdecce5876f0b)) +* **core:** Use class-validator with XSS check for survey answers ([#10490](https://github.com/n8n-io/n8n/issues/10490)) ([547a606](https://github.com/n8n-io/n8n/commit/547a60642ce9e54819d4e600c822d87dabd59b2e)) +* **core:** Use explicit types in configs to ensure valid decorator metadata ([#10433](https://github.com/n8n-io/n8n/issues/10433)) ([2043daa](https://github.com/n8n-io/n8n/commit/2043daa2570bc04b0b8d41f277901a8cc8a7b98f)) +* **editor:** Add workflow scopes when initializing workflow ([#10455](https://github.com/n8n-io/n8n/issues/10455)) ([b857c2c](https://github.com/n8n-io/n8n/commit/b857c2cda0a9e4386a540d5e1e741570d9453588)) +* **editor:** Buffer json chunks in stream response ([#10439](https://github.com/n8n-io/n8n/issues/10439)) ([37797f3](https://github.com/n8n-io/n8n/commit/37797f38d81b12d030ba85034baeb49192ea575c)) +* **editor:** Fix flaky mapping tests ([#10453](https://github.com/n8n-io/n8n/issues/10453)) ([fc6d413](https://github.com/n8n-io/n8n/commit/fc6d4138d58282f676b32f1a6011b1b6d0184bf2)) +* **editor:** Fix overflow in AI Assistant chat messages ([#10491](https://github.com/n8n-io/n8n/issues/10491)) ([4a6ca63](https://github.com/n8n-io/n8n/commit/4a6ca632100731f85875c639f2164bf1ef415009)) +* **editor:** Highlight matching type in filter component ([#10425](https://github.com/n8n-io/n8n/issues/10425)) ([6bca879](https://github.com/n8n-io/n8n/commit/6bca879d4ae30c7f9a35e8d6672de42cf93be727)) +* **editor:** Show item count in output panel schema view ([#10426](https://github.com/n8n-io/n8n/issues/10426)) ([4dee7cc](https://github.com/n8n-io/n8n/commit/4dee7cc36e5f7768d0b71095b194bf357c92e941)) +* **editor:** Truncate long data pill labels in schema view ([#10427](https://github.com/n8n-io/n8n/issues/10427)) ([1bf2f4f](https://github.com/n8n-io/n8n/commit/1bf2f4f6171d666391bb3a3a312468bc083446e3)) +* Filter component - improve errors ([#10456](https://github.com/n8n-io/n8n/issues/10456)) ([61ac0c7](https://github.com/n8n-io/n8n/commit/61ac0c77755210f570b887951fe6bbec1a323811)) +* **Google Sheets Node:** Better error when column to match on is empty ([#10442](https://github.com/n8n-io/n8n/issues/10442)) ([ce46bf5](https://github.com/n8n-io/n8n/commit/ce46bf516a86d9779f37dd75b0c680d26d88e15d)) +* **Google Sheets Node:** Update name and hint for useAppend option ([#10443](https://github.com/n8n-io/n8n/issues/10443)) ([c5a0c04](https://github.com/n8n-io/n8n/commit/c5a0c049eaf44419c690d151de42fb0c10bd406e)) +* **Google Sheets Node:** Update to returnAllMatches option ([#10440](https://github.com/n8n-io/n8n/issues/10440)) ([f7fb02e](https://github.com/n8n-io/n8n/commit/f7fb02e92a756781f8e35bbbfc25d71c12cb70af)) +* **Invoice Ninja Node:** Fix payment types ([#10462](https://github.com/n8n-io/n8n/issues/10462)) ([129245d](https://github.com/n8n-io/n8n/commit/129245da10be1d645f61e929e40b128bd7813f17)) +* **n8n Form Trigger Node:** Show basic authentication modal on wrong credentials ([#10423](https://github.com/n8n-io/n8n/issues/10423)) ([0dc3e99](https://github.com/n8n-io/n8n/commit/0dc3e99b26bec45e747d83f383cfe5169d89e6b7)) +* **OpenAI Node:** Throw node operations error in case of openAi client error ([#10448](https://github.com/n8n-io/n8n/issues/10448)) ([0d3ed46](https://github.com/n8n-io/n8n/commit/0d3ed461996bbad06015c455f133baab6506437f)) +* Project Viewer always seeing a connection error when testing credentials ([#10417](https://github.com/n8n-io/n8n/issues/10417)) ([613cdd2](https://github.com/n8n-io/n8n/commit/613cdd2ba2c0f224c4857a5fc3eea36dbd683049)) +* Remove unimplemented Postgres credentials options ([#10461](https://github.com/n8n-io/n8n/issues/10461)) ([17ac784](https://github.com/n8n-io/n8n/commit/17ac7844f29d819b91dfaf90b9fe386d98060c42)) +* Rename Assistant back ([#10481](https://github.com/n8n-io/n8n/issues/10481)) ([c410aed](https://github.com/n8n-io/n8n/commit/c410aed4c22182943dc80ede63acda00b7898e10)) +* Require mfa code to change email ([#10354](https://github.com/n8n-io/n8n/issues/10354)) ([39c8e50](https://github.com/n8n-io/n8n/commit/39c8e50ad0513649f5a8cef911b7d6cdd61c2372)) +* **Respond to Webhook Node:** Fix issue preventing the chat trigger from working ([#9886](https://github.com/n8n-io/n8n/issues/9886)) ([9d6ad88](https://github.com/n8n-io/n8n/commit/9d6ad88c14a88fd0dfcb4f9981e38d19cf5f3067)) +* Show input names when node has multiple inputs ([#10434](https://github.com/n8n-io/n8n/issues/10434)) ([973956c](https://github.com/n8n-io/n8n/commit/973956cc26c78c329ff6eb6934d4f0a24060c87c)) +* **Toggl Trigger Node:** Update API version ([#10207](https://github.com/n8n-io/n8n/issues/10207)) ([9bdb1d6](https://github.com/n8n-io/n8n/commit/9bdb1d6dca43fe491c5eb96f093b7eec4509eaff)) + + +### Features + +* **core:** Support bidirectional communication between specific mains and specific workers ([#10377](https://github.com/n8n-io/n8n/issues/10377)) ([d0fc9de](https://github.com/n8n-io/n8n/commit/d0fc9dee0e17211c1ed130b19286e9573c9ebfbd)) +* **Facebook Graph API Node:** Update node to support API v18 - v20 ([#10419](https://github.com/n8n-io/n8n/issues/10419)) ([e7ee10f](https://github.com/n8n-io/n8n/commit/e7ee10f243663d899d32e61bc6264b4df444e2af)) + + + +# [1.55.0](https://github.com/n8n-io/n8n/compare/n8n@1.54.0...n8n@1.55.0) (2024-08-14) + + +### Bug Fixes + +* Add better error handling for chat errors ([#10408](https://github.com/n8n-io/n8n/issues/10408)) ([f82b6e4](https://github.com/n8n-io/n8n/commit/f82b6e4ba9bf527b3a4c17872162d9ae124ead0d)) +* **AI Agent Node:** Fix issues with some tools not populating ([#10406](https://github.com/n8n-io/n8n/issues/10406)) ([51a1edd](https://github.com/n8n-io/n8n/commit/51a1eddbf00393f3881c340cf37cfcca59566c99)) +* **core:** Account for cancelling an execution with no workers available ([#10343](https://github.com/n8n-io/n8n/issues/10343)) ([b044e78](https://github.com/n8n-io/n8n/commit/b044e783e73a499dbd7532a5d489a782d3d021da)) +* **core:** Account for owner when filtering by project ID in `GET /workflows` in Public API ([#10379](https://github.com/n8n-io/n8n/issues/10379)) ([5ac65b3](https://github.com/n8n-io/n8n/commit/5ac65b36bcb1351c6233b951f064f60862f790a5)) +* **core:** Enforce shutdown timer and sequence on `SIGINT` for main ([#10346](https://github.com/n8n-io/n8n/issues/10346)) ([5255793](https://github.com/n8n-io/n8n/commit/5255793afee5653d8356b8e4d2e1009d5cf36164)) +* **core:** Filter out prototype and constructor lookups in expressions ([#10382](https://github.com/n8n-io/n8n/issues/10382)) ([8e7d29a](https://github.com/n8n-io/n8n/commit/8e7d29ad3c4872b1cc147dfcfe9a864ba916692f)) +* **core:** Fix duplicate Redis publisher ([#10392](https://github.com/n8n-io/n8n/issues/10392)) ([45813de](https://github.com/n8n-io/n8n/commit/45813debc963096f63cc0aabe82d9d9f853a39d7)) +* **core:** Fix worker shutdown errors when active executions ([#10353](https://github.com/n8n-io/n8n/issues/10353)) ([e071b73](https://github.com/n8n-io/n8n/commit/e071b73bab34edd4b3e6aef6497514acc504cdc6)) +* **core:** Prevent XSS in user update endpoints ([#10338](https://github.com/n8n-io/n8n/issues/10338)) ([7898498](https://github.com/n8n-io/n8n/commit/78984986a6b4add89df9743b94c113046f1d5ee8)) +* **core:** Prevent XSS via static cache dir ([#10339](https://github.com/n8n-io/n8n/issues/10339)) ([4f392b5](https://github.com/n8n-io/n8n/commit/4f392b5e3e0ee166e85a2e060b3ec7fcf145229b)) +* **core:** Rate limit MFA activation and verification endpoints ([#10330](https://github.com/n8n-io/n8n/issues/10330)) ([b6c47c0](https://github.com/n8n-io/n8n/commit/b6c47c0e3214878d42980d5c9535df52b3984b3c)) +* **editor:** Connect up new project viewer role to the FE ([#9913](https://github.com/n8n-io/n8n/issues/9913)) ([117e2d9](https://github.com/n8n-io/n8n/commit/117e2d968fcc535f32c583ac8f2dc8a84e8cd6bd)) +* **editor:** Enable credential sharing between all types of projects ([#10233](https://github.com/n8n-io/n8n/issues/10233)) ([1cf48cc](https://github.com/n8n-io/n8n/commit/1cf48cc3019c1cf27e2f3c9affd18426237e9064)) +* **editor:** Fix rendering of SVG icons in public chat on iOS ([#10381](https://github.com/n8n-io/n8n/issues/10381)) ([7ab3811](https://github.com/n8n-io/n8n/commit/7ab38114dbf3881afba39287a061446ec4bf0431)) +* **editor:** Fixing XSS vulnerability in toast messages ([#10329](https://github.com/n8n-io/n8n/issues/10329)) ([38bdd9f](https://github.com/n8n-io/n8n/commit/38bdd9f5d0d9ca06fab1a7e1a3e7a4a648a6a89a)) +* **editor:** Revert change that hid swagger docs in the ui ([#10350](https://github.com/n8n-io/n8n/issues/10350)) ([bae49d3](https://github.com/n8n-io/n8n/commit/bae49d3198d4bcc27e7996cd4f7be3132becc98e)) +* **n8n Form Trigger Node:** Fix issue preventing v1 node from working ([#10364](https://github.com/n8n-io/n8n/issues/10364)) ([9b647a9](https://github.com/n8n-io/n8n/commit/9b647a9837434e8b75e3ad754ff5136bb5ac760d)) +* Require mfa code for password change if its enabled ([#10341](https://github.com/n8n-io/n8n/issues/10341)) ([9d7caac](https://github.com/n8n-io/n8n/commit/9d7caacc699f10962783393925a980ec6f1ca975)) +* Require mfa code to disable mfa ([#10345](https://github.com/n8n-io/n8n/issues/10345)) ([3384f52](https://github.com/n8n-io/n8n/commit/3384f52a35b835ba1d8633dc94bab0ad6e7023b3)) + + +### Features + +* Add Ask assistant behind feature flag ([#9995](https://github.com/n8n-io/n8n/issues/9995)) ([5ed2a77](https://github.com/n8n-io/n8n/commit/5ed2a77740db1f02b27c571f4dfdfa206ebdb19c)) +* **AI Transform Node:** New node ([#10405](https://github.com/n8n-io/n8n/issues/10405)) ([4d222ac](https://github.com/n8n-io/n8n/commit/4d222ac19d943b69fd9f87abe5e5c5f5141eed8d)) +* **AI Transform Node:** New node ([#9990](https://github.com/n8n-io/n8n/issues/9990)) ([0de9d56](https://github.com/n8n-io/n8n/commit/0de9d56619ed1c055407353046b8a9ebe78da527)) +* **core:** Allow overriding npm registry for community packages ([#10325](https://github.com/n8n-io/n8n/issues/10325)) ([33a2703](https://github.com/n8n-io/n8n/commit/33a2703429d9eaa41f72d2e7d2da5be60b6c620f)) +* **editor:** Add schema view to expression modal ([#9976](https://github.com/n8n-io/n8n/issues/9976)) ([71b6c67](https://github.com/n8n-io/n8n/commit/71b6c671797024d7b516352fa9b7ecda101ce3b2)) +* **MySQL Node:** Return decimal types as numbers ([#10313](https://github.com/n8n-io/n8n/issues/10313)) ([f744d7c](https://github.com/n8n-io/n8n/commit/f744d7c100be68669d9a3efd0033dd371a3cfaf7)) +* **Okta Node:** Add Okta Node ([#10278](https://github.com/n8n-io/n8n/issues/10278)) ([5cac0f3](https://github.com/n8n-io/n8n/commit/5cac0f339d649cfe5857d33738210cbc1599370b)) + + + +# [1.54.0](https://github.com/n8n-io/n8n/compare/n8n@1.53.0...n8n@1.54.0) (2024-08-07) + + +### Bug Fixes + +* **core:** Ensure OAuth token data is not stubbed in source control ([#10302](https://github.com/n8n-io/n8n/issues/10302)) ([98115e9](https://github.com/n8n-io/n8n/commit/98115e95df8289a8ec400a570a7f256382f8e286)) +* **core:** Fix expressions in webhook nodes(Form, Webhook) to access previous node's data ([#10247](https://github.com/n8n-io/n8n/issues/10247)) ([88a1701](https://github.com/n8n-io/n8n/commit/88a170176a3447e7f847e9cf145aeb867b1c5fcf)) +* **core:** Fix user telemetry bugs ([#10293](https://github.com/n8n-io/n8n/issues/10293)) ([42a0b59](https://github.com/n8n-io/n8n/commit/42a0b594d6ea2527c55a2aa9976c904cf70ecf92)) +* **core:** Make execution and its data creation atomic ([#10276](https://github.com/n8n-io/n8n/issues/10276)) ([ae50bb9](https://github.com/n8n-io/n8n/commit/ae50bb95a8e5bf1cdbf9483da54b84094b82e260)) +* **core:** Make OAuth1/OAuth2 callback not require auth ([#10263](https://github.com/n8n-io/n8n/issues/10263)) ([a8e2774](https://github.com/n8n-io/n8n/commit/a8e2774f5382e202556b5506c7788265786aa973)) +* **core:** Revert transactions until we remove the legacy sqlite driver ([#10299](https://github.com/n8n-io/n8n/issues/10299)) ([1eba7c3](https://github.com/n8n-io/n8n/commit/1eba7c3c763ac5b6b28c1c6fc43fc8c215249292)) +* **core:** Surface enterprise trial error message ([#10267](https://github.com/n8n-io/n8n/issues/10267)) ([432ac1d](https://github.com/n8n-io/n8n/commit/432ac1da59e173ce4c0f2abbc416743d9953ba70)) +* **core:** Upgrade tournament to address some XSS vulnerabilities ([#10277](https://github.com/n8n-io/n8n/issues/10277)) ([43ae159](https://github.com/n8n-io/n8n/commit/43ae159ea40c574f8e41bdfd221ab2bf3268eee7)) +* **core:** VM2 sandbox should not throw on `new Promise` ([#10298](https://github.com/n8n-io/n8n/issues/10298)) ([7e95f9e](https://github.com/n8n-io/n8n/commit/7e95f9e2e40a99871f1b6abcdacb39ac5f857332)) +* **core:** Webhook and form baseUrl missing ([#10290](https://github.com/n8n-io/n8n/issues/10290)) ([8131d66](https://github.com/n8n-io/n8n/commit/8131d66f8ca1b1da00597a12859ee4372148a0c9)) +* **editor:** Enable moving resources only if team projects are available by the license ([#10271](https://github.com/n8n-io/n8n/issues/10271)) ([42ba884](https://github.com/n8n-io/n8n/commit/42ba8841c401126c77158a53dc8fcbb45dfce8fd)) +* **editor:** Fix execution retry button ([#10275](https://github.com/n8n-io/n8n/issues/10275)) ([55f2ffe](https://github.com/n8n-io/n8n/commit/55f2ffe256c91a028cee95c3bbb37a093a1c0f81)) +* **editor:** Update design system Avatar component to show initials also when only firstName or lastName is given ([#10308](https://github.com/n8n-io/n8n/issues/10308)) ([46bbf09](https://github.com/n8n-io/n8n/commit/46bbf09beacad12472d91786b91d845fe2afb26d)) +* **editor:** Update tags filter/editor to not show non existing tag as a selectable option ([#10297](https://github.com/n8n-io/n8n/issues/10297)) ([557a76e](https://github.com/n8n-io/n8n/commit/557a76ec2326de72fb7a8b46fc4353f8fd9b591d)) +* **Invoice Ninja Node:** Fix payment types ([#10196](https://github.com/n8n-io/n8n/issues/10196)) ([c5acbb7](https://github.com/n8n-io/n8n/commit/c5acbb7ec0d24ec9b30c221fa3b2fb615fb9ec7f)) +* Loop node no input data shown ([#10224](https://github.com/n8n-io/n8n/issues/10224)) ([c8ee852](https://github.com/n8n-io/n8n/commit/c8ee852159207be0cfe2c3e0ee8e7b29d838aa35)) + + +### Features + +* **core:** Allow filtering executions and users by project in Public API ([#10250](https://github.com/n8n-io/n8n/issues/10250)) ([7056e50](https://github.com/n8n-io/n8n/commit/7056e50b006bda665f64ce6234c5c1967891c415)) +* **core:** Allow transferring credentials in Public API ([#10259](https://github.com/n8n-io/n8n/issues/10259)) ([07d7b24](https://github.com/n8n-io/n8n/commit/07d7b247f02a9d7185beca7817deb779a3d665dd)) +* **core:** Show sub-node error on the logs pane. Open logs pane on sub-node error ([#10248](https://github.com/n8n-io/n8n/issues/10248)) ([57d1c9a](https://github.com/n8n-io/n8n/commit/57d1c9a99e97308f2f1b8ae05ac3861a835e8e5a)) +* **core:** Support community packages in scaling-mode ([#10228](https://github.com/n8n-io/n8n/issues/10228)) ([88086a4](https://github.com/n8n-io/n8n/commit/88086a41ff5b804b35aa9d9503dc2d48836fe4ec)) +* **core:** Support create, delete, edit role for users in Public API ([#10279](https://github.com/n8n-io/n8n/issues/10279)) ([84efbd9](https://github.com/n8n-io/n8n/commit/84efbd9b9c51f536b21a4f969ab607d277bef692)) +* **core:** Support create, read, update, delete projects in Public API ([#10269](https://github.com/n8n-io/n8n/issues/10269)) ([489ce10](https://github.com/n8n-io/n8n/commit/489ce100634c3af678fb300e9a39d273042542e6)) +* **editor:** Auto-add LLM chain for new LLM nodes on empty canvas ([#10245](https://github.com/n8n-io/n8n/issues/10245)) ([06419d9](https://github.com/n8n-io/n8n/commit/06419d9483ae916e79aace6d8c17e265b419b15d)) +* **Elasticsearch Node:** Add bulk operations for Elasticsearch ([#9940](https://github.com/n8n-io/n8n/issues/9940)) ([bf8f848](https://github.com/n8n-io/n8n/commit/bf8f848645dfd31527713a55bd1fc93865327017)) +* **Lemlist Trigger Node:** Update Trigger events ([#10311](https://github.com/n8n-io/n8n/issues/10311)) ([15f10ec](https://github.com/n8n-io/n8n/commit/15f10ec325cb5eda0f952bed3a5f171dd91bc639)) +* **MongoDB Node:** Add projection to query options on Find ([#9972](https://github.com/n8n-io/n8n/issues/9972)) ([0a84e0d](https://github.com/n8n-io/n8n/commit/0a84e0d8b047669f5cf023c21383d01c929c5b4f)) +* **Postgres Chat Memory, Redis Chat Memory, Xata:** Add support for context window length ([#10203](https://github.com/n8n-io/n8n/issues/10203)) ([e3edeaa](https://github.com/n8n-io/n8n/commit/e3edeaa03526f041d15d1099ea91869e38a0decc)) +* **Stripe Trigger Node:** Add Stripe webhook descriptions based on the workflow ID and name ([#9956](https://github.com/n8n-io/n8n/issues/9956)) ([3433465](https://github.com/n8n-io/n8n/commit/34334651e0e6874736a437a894176bed4590e5a7)) +* **Webflow Node:** Update to use the v2 API ([#9996](https://github.com/n8n-io/n8n/issues/9996)) ([6d8323f](https://github.com/n8n-io/n8n/commit/6d8323fadea8af04483eb1a873df0cf3ccc2a891)) + + + +# [1.53.0](https://github.com/n8n-io/n8n/compare/n8n@1.52.0...n8n@1.53.0) (2024-07-31) + + +### Bug Fixes + +* Better error message when calling data transformation functions on a null value ([#10210](https://github.com/n8n-io/n8n/issues/10210)) ([1718125](https://github.com/n8n-io/n8n/commit/1718125c6d8589cf24dc8d34f6808dd6f1802691)) +* **core:** Fix missing successful items on continueErrorOutput with multiple outputs ([#10218](https://github.com/n8n-io/n8n/issues/10218)) ([1a7713e](https://github.com/n8n-io/n8n/commit/1a7713ef263680da43f08b6c8a15aee7a0341493)) +* **core:** Flush instance stopped event immediately ([#10238](https://github.com/n8n-io/n8n/issues/10238)) ([d6770b5](https://github.com/n8n-io/n8n/commit/d6770b5fcaec6438d677b918aaeb1669ad7424c2)) +* **core:** Restore log event `n8n.workflow.failed` ([#10253](https://github.com/n8n-io/n8n/issues/10253)) ([3e96b29](https://github.com/n8n-io/n8n/commit/3e96b293329525c9d4b2fcef87b3803e458c8e7f)) +* **core:** Upgrade @n8n/vm2 to address CVE‑2023‑37466 ([#10265](https://github.com/n8n-io/n8n/issues/10265)) ([2a09a03](https://github.com/n8n-io/n8n/commit/2a09a036d2e916acff7ee50904f1d011a93758e1)) +* **editor:** Defer `User saved credentials` telemetry event for OAuth credentials ([#10215](https://github.com/n8n-io/n8n/issues/10215)) ([40a5226](https://github.com/n8n-io/n8n/commit/40a5226e24448a4428143e69d80ebc78238365a1)) +* **editor:** Fix custom API call notice ([#10227](https://github.com/n8n-io/n8n/issues/10227)) ([5b47c8b](https://github.com/n8n-io/n8n/commit/5b47c8b57b25528cd2d6f97bc6d98707d47f35bc)) +* **editor:** Fix issue with existing credential not opening in HTTP agent tool ([#10167](https://github.com/n8n-io/n8n/issues/10167)) ([906b4c3](https://github.com/n8n-io/n8n/commit/906b4c3c7b2919111cf23eaa12b3c4d507969179)) +* **editor:** Fix parameter input glitch when there was an error loading remote options ([#10209](https://github.com/n8n-io/n8n/issues/10209)) ([c0e3743](https://github.com/n8n-io/n8n/commit/c0e37439a87105a0e66c8ebced42c06dab30dc5e)) +* **editor:** Fix workflow execution list scrolling after filter change ([#10226](https://github.com/n8n-io/n8n/issues/10226)) ([7e64358](https://github.com/n8n-io/n8n/commit/7e643589c67adc0218216ec4b89a95f0edfedbee)) +* **Google BigQuery Node:** Send timeoutMs in query, pagination support ([#10205](https://github.com/n8n-io/n8n/issues/10205)) ([f5722e8](https://github.com/n8n-io/n8n/commit/f5722e8823ccd2bc2b5f43ba3c849797d5690a93)) +* **Google Sheets Node:** Add column names row if sheet is empty ([#10200](https://github.com/n8n-io/n8n/issues/10200)) ([82eba9f](https://github.com/n8n-io/n8n/commit/82eba9fc5ff49b8e2a9db93c10b253fb67a8c644)) +* **Google Sheets Node:** Do not insert row_number as a new column, do not checkForSchemaChanges in update operation ([#10201](https://github.com/n8n-io/n8n/issues/10201)) ([5136d10](https://github.com/n8n-io/n8n/commit/5136d10ca3492f92af67d4a1d4abc774419580cc)) +* **Google Sheets Node:** Fix Google Sheet URL regex ([#10195](https://github.com/n8n-io/n8n/issues/10195)) ([e6fd996](https://github.com/n8n-io/n8n/commit/e6fd996973d4f40facf0ebf1eea3cc26acd0603d)) +* **HTTP Request Node:** Resolve max pages expression ([#10192](https://github.com/n8n-io/n8n/issues/10192)) ([bfc8e1b](https://github.com/n8n-io/n8n/commit/bfc8e1b56f7714e1f52aae747d58d686b86e60f0)) +* **LinkedIn Node:** Fix issue with some characters cutting off posts early ([#10185](https://github.com/n8n-io/n8n/issues/10185)) ([361b5e7](https://github.com/n8n-io/n8n/commit/361b5e7c37ba49b68dcf5b8122621aad4d8d96e0)) +* **Postgres Node:** Expressions in query parameters for Postgres executeQuery operation ([#10217](https://github.com/n8n-io/n8n/issues/10217)) ([519fc4d](https://github.com/n8n-io/n8n/commit/519fc4d75325a80b84cc4dcacf52d6f4c02e3a44)) +* **Postgres Node:** Option to treat query parameters enclosed in single quotas as text ([#10214](https://github.com/n8n-io/n8n/issues/10214)) ([00ec253](https://github.com/n8n-io/n8n/commit/00ec2533374d3def465efee718592fc4001d5602)) +* **Read/Write Files from Disk Node:** Notice update in file selector, replace backslashes with forward slashes if windows path ([#10186](https://github.com/n8n-io/n8n/issues/10186)) ([3eac673](https://github.com/n8n-io/n8n/commit/3eac673b17986c5c74bd2adb5ad589ba0ca55319)) +* **Text Classifier Node:** Use proper documentation URL and respect continueOnFail ([#10216](https://github.com/n8n-io/n8n/issues/10216)) ([452f52c](https://github.com/n8n-io/n8n/commit/452f52c124017e002e86c547ba42b1633b14beed)) +* **Trello Node:** Use body for POST requests ([#10189](https://github.com/n8n-io/n8n/issues/10189)) ([7775d50](https://github.com/n8n-io/n8n/commit/7775d5059b7f69d9af22e7ad7d12c6cf9092a4e5)) +* **Wait Node:** Authentication fix ([#10236](https://github.com/n8n-io/n8n/issues/10236)) ([f87854f](https://github.com/n8n-io/n8n/commit/f87854f8db360b7b870583753fcfb4af95adab8c)) + + +### Features + +* **Calendly Trigger Node:** Add OAuth Credentials Support ([#10251](https://github.com/n8n-io/n8n/issues/10251)) ([326c983](https://github.com/n8n-io/n8n/commit/326c983915a2c382e32398358e7dcadd022c0b77)) +* **core:** Allow filtering workflows by project and transferring workflows in Public API ([#10231](https://github.com/n8n-io/n8n/issues/10231)) ([d719899](https://github.com/n8n-io/n8n/commit/d719899223907b20a17883a35e4ef637a3453532)) +* **editor:** Show new executions as `Queued` in the UI, until they actually start ([#10204](https://github.com/n8n-io/n8n/issues/10204)) ([44728d7](https://github.com/n8n-io/n8n/commit/44728d72423f5549dda09589f4a618ebd80899cb)) +* **HTTP Request Node:** Add option to disable lowercase headers ([#10154](https://github.com/n8n-io/n8n/issues/10154)) ([5aba69b](https://github.com/n8n-io/n8n/commit/5aba69bcf4d232d9860f3cd9fe57cb8839a2f96f)) +* **Information Extractor Node:** Add new simplified AI-node for information extraction ([#10149](https://github.com/n8n-io/n8n/issues/10149)) ([3d235b0](https://github.com/n8n-io/n8n/commit/3d235b0b2df756df35ac60e3dcd87ad183a07167)) +* Introduce Google Cloud Platform as external secrets provider ([#10146](https://github.com/n8n-io/n8n/issues/10146)) ([3ccb9df](https://github.com/n8n-io/n8n/commit/3ccb9df2f902e46f8cbb9c46c0727f29d752a773)) +* **n8n Form Trigger Node:** Improvements ([#10092](https://github.com/n8n-io/n8n/issues/10092)) ([711b667](https://github.com/n8n-io/n8n/commit/711b667ebefe55740e5eb39f1f0f24ceee10e7b0)) +* Recovery option for jsonParse helper ([#10182](https://github.com/n8n-io/n8n/issues/10182)) ([d165b33](https://github.com/n8n-io/n8n/commit/d165b33ceac4d24d0fc290bffe63b5f551204e38)) +* **Sentiment Analysis Node:** Implement Sentiment Analysis node ([#10184](https://github.com/n8n-io/n8n/issues/10184)) ([8ef0a0c](https://github.com/n8n-io/n8n/commit/8ef0a0c58ac2a84aad649ccbe72aa907d005cc44)) +* **Shopify Node:** Update Shopify API version ([#10155](https://github.com/n8n-io/n8n/issues/10155)) ([e2ee915](https://github.com/n8n-io/n8n/commit/e2ee91569a382bfbf787cf45204c72c821a860a0)) +* Support create, read, delete variables in Public API ([#10241](https://github.com/n8n-io/n8n/issues/10241)) ([af695eb](https://github.com/n8n-io/n8n/commit/af695ebf934526d926ea87fe87df61aa73d70979)) + + + # [1.52.0](https://github.com/n8n-io/n8n/compare/n8n@1.51.0...n8n@1.52.0) (2024-07-24) diff --git a/README.md b/README.md index 145ecab8c6..d51ac596ca 100644 --- a/README.md +++ b/README.md @@ -95,8 +95,8 @@ development environment ready in minutes. ## License n8n is [fair-code](https://faircode.io) distributed under the -[**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md) and the -[**n8n Enterprise License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE_EE.md). +[**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/LICENSE.md) and the +[**n8n Enterprise License**](https://github.com/n8n-io/n8n/blob/master/LICENSE_EE.md). Proprietary licenses are available for enterprise customers. [Get in touch](mailto:license@n8n.io) diff --git a/cypress/composables/modals/save-changes-modal.ts b/cypress/composables/modals/save-changes-modal.ts new file mode 100644 index 0000000000..d44b09bd46 --- /dev/null +++ b/cypress/composables/modals/save-changes-modal.ts @@ -0,0 +1,3 @@ +export function getSaveChangesModal() { + return cy.get('.el-overlay').contains('Save changes before leaving?'); +} diff --git a/cypress/composables/projects.ts b/cypress/composables/projects.ts index 6fa2c6f502..84379088d1 100644 --- a/cypress/composables/projects.ts +++ b/cypress/composables/projects.ts @@ -1,41 +1,45 @@ import { CredentialsModal, WorkflowPage } from '../pages'; +import { getVisibleSelect } from '../utils'; const workflowPage = new WorkflowPage(); const credentialsModal = new CredentialsModal(); export const getHomeButton = () => cy.getByTestId('project-home-menu-item'); export const getMenuItems = () => cy.getByTestId('project-menu-item'); -export const getAddProjectButton = () => cy.getByTestId('add-project-menu-item'); +export const getAddProjectButton = () => + cy.getByTestId('add-project-menu-item').should('contain', 'Add project').should('be.visible'); export const getProjectTabs = () => cy.getByTestId('project-tabs').find('a'); export const getProjectTabWorkflows = () => getProjectTabs().filter('a[href$="/workflows"]'); export const getProjectTabCredentials = () => getProjectTabs().filter('a[href$="/credentials"]'); export const getProjectTabSettings = () => getProjectTabs().filter('a[href$="/settings"]'); -export const getProjectSettingsNameInput = () => cy.getByTestId('project-settings-name-input'); +export const getProjectSettingsNameInput = () => + cy.getByTestId('project-settings-name-input').find('input'); export const getProjectSettingsSaveButton = () => cy.getByTestId('project-settings-save-button'); export const getProjectSettingsCancelButton = () => cy.getByTestId('project-settings-cancel-button'); export const getProjectSettingsDeleteButton = () => cy.getByTestId('project-settings-delete-button'); export const getProjectMembersSelect = () => cy.getByTestId('project-members-select'); -export const addProjectMember = (email: string) => { +export const addProjectMember = (email: string, role?: string) => { getProjectMembersSelect().click(); getProjectMembersSelect().get('.el-select-dropdown__item').contains(email.toLowerCase()).click(); + + if (role) { + cy.getByTestId(`user-list-item-${email}`) + .find('[data-test-id="projects-settings-user-role-select"]') + .click(); + getVisibleSelect().find('li').contains(role).click(); + } }; -export const getProjectNameInput = () => cy.get('#projectName').find('input'); export const getResourceMoveModal = () => cy.getByTestId('project-move-resource-modal'); export const getResourceMoveConfirmModal = () => cy.getByTestId('project-move-resource-confirm-modal'); export const getProjectMoveSelect = () => cy.getByTestId('project-move-resource-modal-select'); export function createProject(name: string) { - getAddProjectButton().should('be.visible').click(); + getAddProjectButton().click(); - getProjectNameInput() - .should('be.visible') - .should('be.focused') - .should('have.value', 'My project') - .clear() - .type(name); + getProjectSettingsNameInput().should('be.visible').clear().type(name); getProjectSettingsSaveButton().click(); } @@ -46,7 +50,7 @@ export function createWorkflow(fixtureKey: string, name: string) { workflowPage.actions.zoomToFit(); } -export function createCredential(name: string) { +export function createCredential(name: string, closeModal = true) { credentialsModal.getters.newCredentialModal().should('be.visible'); credentialsModal.getters.newCredentialTypeSelect().should('be.visible'); credentialsModal.getters.newCredentialTypeOption('Notion API').click(); @@ -54,13 +58,8 @@ export function createCredential(name: string) { credentialsModal.getters.connectionParameter('Internal Integration Secret').type('1234567890'); credentialsModal.actions.setName(name); credentialsModal.actions.save(); - credentialsModal.actions.close(); -} -export const actions = { - createProject: (name: string) => { - getAddProjectButton().click(); - getProjectSettingsNameInput().type(name); - getProjectSettingsSaveButton().click(); - }, -}; + if (closeModal) { + credentialsModal.actions.close(); + } +} diff --git a/cypress/constants.ts b/cypress/constants.ts index 8439952ac7..6f7e7b978d 100644 --- a/cypress/constants.ts +++ b/cypress/constants.ts @@ -37,6 +37,7 @@ export const INSTANCE_MEMBERS = [ export const MANUAL_TRIGGER_NODE_NAME = 'Manual Trigger'; export const MANUAL_TRIGGER_NODE_DISPLAY_NAME = 'When clicking ‘Test workflow’'; export const MANUAL_CHAT_TRIGGER_NODE_NAME = 'Chat Trigger'; +export const CHAT_TRIGGER_NODE_DISPLAY_NAME = 'When chat message received'; export const SCHEDULE_TRIGGER_NODE_NAME = 'Schedule Trigger'; export const CODE_NODE_NAME = 'Code'; export const SET_NODE_NAME = 'Set'; @@ -57,6 +58,7 @@ export const AI_TOOL_CODE_NODE_NAME = 'Code Tool'; export const AI_TOOL_WIKIPEDIA_NODE_NAME = 'Wikipedia'; export const AI_TOOL_HTTP_NODE_NAME = 'HTTP Request Tool'; export const AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME = 'OpenAI Chat Model'; +export const AI_MEMORY_POSTGRES_NODE_NAME = 'Postgres Chat Memory'; export const AI_OUTPUT_PARSER_AUTO_FIXING_NODE_NAME = 'Auto-fixing Output Parser'; export const WEBHOOK_NODE_NAME = 'Webhook'; diff --git a/cypress/e2e/11-inline-expression-editor.cy.ts b/cypress/e2e/11-inline-expression-editor.cy.ts index 143648ce1b..945c62821b 100644 --- a/cypress/e2e/11-inline-expression-editor.cy.ts +++ b/cypress/e2e/11-inline-expression-editor.cy.ts @@ -11,6 +11,21 @@ describe('Inline expression editor', () => { cy.on('uncaught:exception', (error) => error.name !== 'ExpressionError'); }); + describe('Basic UI functionality', () => { + it('should open and close inline expression preview', () => { + WorkflowPage.actions.zoomToFit(); + WorkflowPage.actions.openNode('Schedule'); + WorkflowPage.actions.openInlineExpressionEditor(); + WorkflowPage.getters.inlineExpressionEditorInput().clear(); + WorkflowPage.getters.inlineExpressionEditorInput().click().type('{{'); + WorkflowPage.getters.inlineExpressionEditorInput().type('123'); + WorkflowPage.getters.inlineExpressionEditorOutput().contains(/^123$/); + // click outside to close + ndv.getters.outputPanel().click(); + WorkflowPage.getters.inlineExpressionEditorOutput().should('not.exist'); + }); + }); + describe('Static data', () => { beforeEach(() => { WorkflowPage.actions.addNodeToCanvas('Hacker News'); diff --git a/cypress/e2e/12-canvas-actions.cy.ts b/cypress/e2e/12-canvas-actions.cy.ts index 9b05cb84d4..53dad1cc89 100644 --- a/cypress/e2e/12-canvas-actions.cy.ts +++ b/cypress/e2e/12-canvas-actions.cy.ts @@ -145,7 +145,16 @@ describe('Canvas Actions', () => { }); }); - it('should delete connections by pressing the delete button', () => { + it('should delete node by pressing keyboard backspace', () => { + WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); + WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); + WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); + WorkflowPage.getters.canvasNodeByName(CODE_NODE_NAME).click(); + cy.get('body').type('{backspace}'); + WorkflowPage.getters.nodeConnections().should('have.length', 0); + }); + + it('should delete connections by clicking on the delete button', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); diff --git a/cypress/e2e/14-mapping.cy.ts b/cypress/e2e/14-mapping.cy.ts index 2a33aee5c0..43103415e3 100644 --- a/cypress/e2e/14-mapping.cy.ts +++ b/cypress/e2e/14-mapping.cy.ts @@ -40,11 +40,13 @@ describe('Data mapping', () => { ndv.actions.mapDataFromHeader(1, 'value'); ndv.getters.inlineExpressionEditorInput().should('have.text', '{{ $json.timestamp }}'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); + ndv.getters.parameterExpressionPreview('value').should('include.text', '2024'); ndv.actions.mapDataFromHeader(2, 'value'); ndv.getters .inlineExpressionEditorInput() - .should('have.text', "{{ $json.timestamp }} {{ $json['Readable date'] }}"); + .should('have.text', "{{ $json['Readable date'] }}{{ $json.timestamp }}"); }); it('maps expressions from table json, and resolves value based on hover', () => { @@ -133,6 +135,7 @@ describe('Data mapping', () => { ndv.actions.mapToParameter('value'); ndv.getters.inlineExpressionEditorInput().should('have.text', '{{ $json.input[0].count }}'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); ndv.getters.parameterExpressionPreview('value').should('include.text', '0'); ndv.getters @@ -145,8 +148,9 @@ describe('Data mapping', () => { ndv.actions.mapToParameter('value'); ndv.getters .inlineExpressionEditorInput() - .should('have.text', '{{ $json.input[0].count }} {{ $json.input }}'); - ndv.actions.validateExpressionPreview('value', '0 [object Object]'); + .should('have.text', '{{ $json.input }}{{ $json.input[0].count }}'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); + ndv.actions.validateExpressionPreview('value', '[object Object]0'); }); it('maps expressions from schema view', () => { @@ -163,6 +167,7 @@ describe('Data mapping', () => { ndv.actions.mapToParameter('value'); ndv.getters.inlineExpressionEditorInput().should('have.text', '{{ $json.input[0].count }}'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); ndv.actions.validateExpressionPreview('value', '0'); ndv.getters.inputDataContainer().find('span').contains('input').realMouseDown(); @@ -170,8 +175,8 @@ describe('Data mapping', () => { ndv.actions.mapToParameter('value'); ndv.getters .inlineExpressionEditorInput() - .should('have.text', '{{ $json.input[0].count }} {{ $json.input }}'); - ndv.actions.validateExpressionPreview('value', '0 [object Object]'); + .should('have.text', '{{ $json.input }}{{ $json.input[0].count }}'); + ndv.actions.validateExpressionPreview('value', '[object Object]0'); }); it('maps expressions from previous nodes', () => { @@ -192,6 +197,7 @@ describe('Data mapping', () => { ndv.getters .inlineExpressionEditorInput() .should('have.text', `{{ $('${SCHEDULE_TRIGGER_NODE_NAME}').item.json.input[0].count }}`); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); ndv.actions.switchInputMode('Table'); ndv.actions.selectInputNode(SCHEDULE_TRIGGER_NODE_NAME); @@ -200,17 +206,17 @@ describe('Data mapping', () => { .inlineExpressionEditorInput() .should( 'have.text', - `{{ $('${SCHEDULE_TRIGGER_NODE_NAME}').item.json.input[0].count }} {{ $('${SCHEDULE_TRIGGER_NODE_NAME}').item.json.input }}`, + `{{ $('${SCHEDULE_TRIGGER_NODE_NAME}').item.json.input }}{{ $('${SCHEDULE_TRIGGER_NODE_NAME}').item.json.input[0].count }}`, ); ndv.actions.selectInputNode('Set'); ndv.getters.executingLoader().should('not.exist'); ndv.getters.inputDataContainer().should('exist'); - ndv.actions.validateExpressionPreview('value', '0 [object Object]'); + ndv.actions.validateExpressionPreview('value', '[object Object]0'); ndv.getters.inputTbodyCell(2, 0).realHover(); - ndv.actions.validateExpressionPreview('value', '1 [object Object]'); + ndv.actions.validateExpressionPreview('value', '[object Object]1'); }); it('maps keys to path', () => { @@ -271,12 +277,12 @@ describe('Data mapping', () => { ndv.actions.typeIntoParameterInput('value', 'fun'); ndv.actions.clearParameterInput('value'); // keep focus on param - cy.wait(300); ndv.getters.inputDataContainer().should('exist').find('span').contains('count').realMouseDown(); ndv.actions.mapToParameter('value'); ndv.getters.inlineExpressionEditorInput().should('have.text', '{{ $json.input[0].count }}'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); ndv.actions.validateExpressionPreview('value', '0'); ndv.getters.inputDataContainer().find('span').contains('input').realMouseDown(); @@ -284,8 +290,8 @@ describe('Data mapping', () => { ndv.actions.mapToParameter('value'); ndv.getters .inlineExpressionEditorInput() - .should('have.text', '{{ $json.input[0].count }} {{ $json.input }}'); - ndv.actions.validateExpressionPreview('value', '0 [object Object]'); + .should('have.text', '{{ $json.input }}{{ $json.input[0].count }}'); + ndv.actions.validateExpressionPreview('value', '[object Object]0'); }); it('renders expression preview when a previous node is selected', () => { @@ -342,4 +348,31 @@ describe('Data mapping', () => { .invoke('css', 'border') .should('include', 'dashed rgb(90, 76, 194)'); }); + + it('maps expressions to a specific location in the editor', () => { + cy.fixture('Test_workflow_3.json').then((data) => { + cy.get('body').paste(JSON.stringify(data)); + }); + workflowPage.actions.zoomToFit(); + + workflowPage.actions.openNode('Set'); + ndv.actions.typeIntoParameterInput('value', '='); + ndv.getters.inlineExpressionEditorInput().find('.cm-content').paste('hello world\n\nnewline'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); + + ndv.getters.inputDataContainer().should('exist').find('span').contains('count').realMouseDown(); + ndv.actions.mapToParameter('value'); + ndv.getters + .inlineExpressionEditorInput() + .should('have.text', '{{ $json.input[0].count }}hello worldnewline'); + ndv.getters.inlineExpressionEditorInput().type('{esc}'); + ndv.actions.validateExpressionPreview('value', '0hello world\n\nnewline'); + + ndv.getters.inputDataContainer().find('span').contains('input').realMouseDown(); + ndv.actions.mapToParameter('value', 'center'); + + ndv.getters + .inlineExpressionEditorInput() + .should('have.text', '{{ $json.input[0].count }}hello world{{ $json.input }}newline'); + }); }); diff --git a/cypress/e2e/17-sharing.cy.ts b/cypress/e2e/17-sharing.cy.ts index 54c5e6efe2..64769ae193 100644 --- a/cypress/e2e/17-sharing.cy.ts +++ b/cypress/e2e/17-sharing.cy.ts @@ -7,7 +7,7 @@ import { WorkflowSharingModal, WorkflowsPage, } from '../pages'; -import { getVisibleSelect } from '../utils'; +import { getVisibleDropdown, getVisibleSelect } from '../utils'; import * as projects from '../composables/projects'; /** @@ -192,11 +192,79 @@ describe('Sharing', { disableAutoLogin: true }, () => { credentialsModal.actions.saveSharing(); credentialsModal.actions.close(); }); + + it('credentials should work between team and personal projects', () => { + cy.resetDatabase(); + cy.enableFeature('sharing'); + cy.enableFeature('advancedPermissions'); + cy.enableFeature('projectRole:admin'); + cy.enableFeature('projectRole:editor'); + cy.changeQuota('maxTeamProjects', -1); + + cy.signinAsOwner(); + cy.visit('/'); + + projects.createProject('Development'); + + projects.getHomeButton().click(); + workflowsPage.getters.newWorkflowButtonCard().click(); + projects.createWorkflow('Test_workflow_1.json', 'Test workflow'); + + projects.getHomeButton().click(); + projects.getProjectTabCredentials().click(); + credentialsPage.getters.emptyListCreateCredentialButton().click(); + projects.createCredential('Notion API'); + + credentialsPage.getters.credentialCard('Notion API').click(); + credentialsModal.actions.changeTab('Sharing'); + credentialsModal.getters.usersSelect().click(); + getVisibleSelect() + .find('li') + .should('have.length', 4) + .filter(':contains("Development")') + .should('have.length', 1) + .click(); + credentialsModal.getters.saveButton().click(); + credentialsModal.actions.close(); + + projects.getProjectTabWorkflows().click(); + workflowsPage.getters.workflowCardActions('Test workflow').click(); + getVisibleDropdown().find('li').contains('Share').click(); + + workflowSharingModal.getters.usersSelect().filter(':visible').click(); + getVisibleSelect().find('li').should('have.length', 3).first().click(); + workflowSharingModal.getters.saveButton().click(); + + projects.getMenuItems().first().click(); + workflowsPage.getters.newWorkflowButtonCard().click(); + projects.createWorkflow('Test_workflow_1.json', 'Test workflow 2'); + workflowPage.actions.openShareModal(); + workflowSharingModal.getters.usersSelect().should('not.exist'); + + cy.get('body').type('{esc}'); + + projects.getMenuItems().first().click(); + projects.getProjectTabCredentials().click(); + credentialsPage.getters.createCredentialButton().click(); + projects.createCredential('Notion API 2', false); + credentialsModal.actions.changeTab('Sharing'); + credentialsModal.getters.usersSelect().click(); + getVisibleSelect().find('li').should('have.length', 4).first().click(); + credentialsModal.getters.saveButton().click(); + credentialsModal.actions.close(); + + credentialsPage.getters + .credentialCards() + .should('have.length', 2) + .filter(':contains("Owned by me")') + .should('have.length', 1); + }); }); describe('Credential Usage in Cross Shared Workflows', () => { beforeEach(() => { cy.resetDatabase(); + cy.enableFeature('sharing'); cy.enableFeature('advancedPermissions'); cy.enableFeature('projectRole:admin'); cy.enableFeature('projectRole:editor'); @@ -207,23 +275,18 @@ describe('Credential Usage in Cross Shared Workflows', () => { }); it('should only show credentials from the same team project', () => { - cy.enableFeature('advancedPermissions'); - cy.enableFeature('projectRole:admin'); - cy.enableFeature('projectRole:editor'); - cy.changeQuota('maxTeamProjects', -1); - // Create a notion credential in the home project credentialsPage.getters.emptyListCreateCredentialButton().click(); credentialsModal.actions.createNewCredential('Notion API'); // Create a notion credential in one project - projects.actions.createProject('Development'); + projects.createProject('Development'); projects.getProjectTabCredentials().click(); credentialsPage.getters.emptyListCreateCredentialButton().click(); credentialsModal.actions.createNewCredential('Notion API'); // Create a notion credential in another project - projects.actions.createProject('Test'); + projects.createProject('Test'); projects.getProjectTabCredentials().click(); credentialsPage.getters.emptyListCreateCredentialButton().click(); credentialsModal.actions.createNewCredential('Notion API'); @@ -238,10 +301,36 @@ describe('Credential Usage in Cross Shared Workflows', () => { getVisibleSelect().find('li').should('have.length', 2); }); + it('should only show credentials in their personal project for members', () => { + // Create a notion credential as the owner + credentialsPage.getters.emptyListCreateCredentialButton().click(); + credentialsModal.actions.createNewCredential('Notion API'); + + // Create another notion credential as the owner, but share it with member + // 0 + credentialsPage.getters.createCredentialButton().click(); + credentialsModal.actions.createNewCredential('Notion API', false); + credentialsModal.actions.changeTab('Sharing'); + credentialsModal.actions.addUser(INSTANCE_MEMBERS[0].email); + credentialsModal.actions.saveSharing(); + + // As the member, create a new notion credential and a workflow + cy.signinAsMember(); + cy.visit(credentialsPage.url); + credentialsPage.getters.createCredentialButton().click(); + credentialsModal.actions.createNewCredential('Notion API'); + cy.visit(workflowsPage.url); + workflowsPage.actions.createWorkflowFromCard(); + workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); + + // Only the own credential the shared one (+ the 'Create new' option) + // should be in the dropdown + workflowPage.getters.nodeCredentialsSelect().click(); + getVisibleSelect().find('li').should('have.length', 3); + }); + it('should only show credentials in their personal project for members if the workflow was shared with them', () => { const workflowName = 'Test workflow'; - cy.enableFeature('sharing'); - cy.reload(); // Create a notion credential as the owner and a workflow that is shared // with member 0 @@ -272,7 +361,6 @@ describe('Credential Usage in Cross Shared Workflows', () => { it("should show all credentials from all personal projects the workflow's been shared into for the global owner", () => { const workflowName = 'Test workflow'; - cy.enableFeature('sharing'); // As member 1, create a new notion credential. This should not show up. cy.signinAsMember(1); @@ -317,8 +405,6 @@ describe('Credential Usage in Cross Shared Workflows', () => { }); it('should show all personal credentials if the global owner owns the workflow', () => { - cy.enableFeature('sharing'); - // As member 0, create a new notion credential. cy.signinAsMember(); cy.visit(credentialsPage.url); diff --git a/cypress/e2e/17-workflow-tags.cy.ts b/cypress/e2e/17-workflow-tags.cy.ts index 88a2c9973d..26ea7cbe2c 100644 --- a/cypress/e2e/17-workflow-tags.cy.ts +++ b/cypress/e2e/17-workflow-tags.cy.ts @@ -1,4 +1,5 @@ import { WorkflowPage } from '../pages'; +import { getVisibleSelect } from '../utils'; const wf = new WorkflowPage(); @@ -64,10 +65,26 @@ describe('Workflow tags', () => { it('should detach a tag inline by clicking on dropdown list item', () => { wf.getters.createTagButton().click(); wf.actions.addTags(TEST_TAGS); - wf.getters.nthTagPill(1).click(); + wf.getters.workflowTagsContainer().click(); wf.getters.tagsInDropdown().filter('.selected').first().click(); cy.get('body').click(0, 0); wf.getters.workflowTags().click(); wf.getters.tagPills().should('have.length', TEST_TAGS.length - 1); }); + + it('should not show non existing tag as a selectable option', () => { + const NON_EXISTING_TAG = 'My Test Tag'; + + wf.getters.createTagButton().click(); + wf.actions.addTags(TEST_TAGS); + cy.get('body').click(0, 0); + wf.getters.workflowTags().click(); + wf.getters.workflowTagsInput().type(NON_EXISTING_TAG); + + getVisibleSelect() + .find('li') + .should('have.length', 2) + .filter(`:contains("${NON_EXISTING_TAG}")`) + .should('not.have.length'); + }); }); diff --git a/cypress/e2e/1858-PAY-can-use-context-menu.ts b/cypress/e2e/1858-PAY-can-use-context-menu.ts new file mode 100644 index 0000000000..6727df4166 --- /dev/null +++ b/cypress/e2e/1858-PAY-can-use-context-menu.ts @@ -0,0 +1,21 @@ +import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; + +const WorkflowPage = new WorkflowPageClass(); + +describe('PAY-1858 context menu', () => { + it('can use context menu on saved workflow', () => { + WorkflowPage.actions.visit(); + cy.createFixtureWorkflow('Test_workflow_filter.json', 'test'); + + WorkflowPage.getters.canvasNodes().should('have.length', 5); + WorkflowPage.actions.deleteNodeFromContextMenu('Then'); + WorkflowPage.getters.canvasNodes().should('have.length', 4); + + WorkflowPage.actions.hitSaveWorkflow(); + + cy.reload(); + WorkflowPage.getters.canvasNodes().should('have.length', 4); + WorkflowPage.actions.deleteNodeFromContextMenu('Code'); + WorkflowPage.getters.canvasNodes().should('have.length', 3); + }); +}); diff --git a/cypress/e2e/19-execution.cy.ts b/cypress/e2e/19-execution.cy.ts index 804e81d4e6..1f7d5c332e 100644 --- a/cypress/e2e/19-execution.cy.ts +++ b/cypress/e2e/19-execution.cy.ts @@ -275,7 +275,6 @@ describe('Execution', () => { .within(() => cy.get('.fa-check').should('not.exist')); successToast().should('be.visible'); - clearNotifications(); // Clear execution data workflowPage.getters.clearExecutionDataButton().should('be.visible'); diff --git a/cypress/e2e/2-credentials.cy.ts b/cypress/e2e/2-credentials.cy.ts index ffecd51959..9dbe6c6b5d 100644 --- a/cypress/e2e/2-credentials.cy.ts +++ b/cypress/e2e/2-credentials.cy.ts @@ -112,13 +112,13 @@ describe('Credentials', () => { workflowPage.getters.nodeCredentialsSelect().should('have.length', 2); workflowPage.getters.nodeCredentialsSelect().first().click(); - getVisibleSelect().find('li').last().click(); + getVisibleSelect().find('li').contains('Create New Credential').click(); // This one should show auth type selector credentialsModal.getters.credentialAuthTypeRadioButtons().should('have.length', 2); cy.get('body').type('{esc}'); workflowPage.getters.nodeCredentialsSelect().last().click(); - getVisibleSelect().find('li').last().click(); + getVisibleSelect().find('li').contains('Create New Credential').click(); // This one should not show auth type selector credentialsModal.getters.credentialsAuthTypeSelector().should('not.exist'); }); diff --git a/cypress/e2e/20-workflow-executions.cy.ts b/cypress/e2e/20-workflow-executions.cy.ts index 075923e940..59f08c570b 100644 --- a/cypress/e2e/20-workflow-executions.cy.ts +++ b/cypress/e2e/20-workflow-executions.cy.ts @@ -183,6 +183,50 @@ describe('Current Workflow Executions', () => { .invoke('attr', 'title') .should('eq', newWorkflowName); }); + + it('should load items and auto scroll after filter change', () => { + createMockExecutions(); + createMockExecutions(); + cy.intercept('GET', '/rest/executions?filter=*').as('getExecutions'); + + executionsTab.actions.switchToExecutionsTab(); + + cy.wait(['@getExecutions']); + + executionsTab.getters.executionsList().scrollTo(0, 500).wait(0); + + executionsTab.getters.executionListItems().eq(10).click(); + + cy.getByTestId('executions-filter-button').click(); + cy.getByTestId('executions-filter-status-select').should('be.visible').click(); + getVisibleSelect().find('li:contains("Error")').click(); + + executionsTab.getters.executionListItems().should('have.length', 5); + executionsTab.getters.successfulExecutionListItems().should('have.length', 1); + executionsTab.getters.failedExecutionListItems().should('have.length', 4); + + cy.getByTestId('executions-filter-button').click(); + cy.getByTestId('executions-filter-status-select').should('be.visible').click(); + getVisibleSelect().find('li:contains("Success")').click(); + + // check if the list is scrolled + executionsTab.getters.executionListItems().eq(10).should('be.visible'); + executionsTab.getters.executionsList().then(($el) => { + const { scrollTop, scrollHeight, clientHeight } = $el[0]; + expect(scrollTop).to.be.greaterThan(0); + expect(scrollTop + clientHeight).to.be.lessThan(scrollHeight); + + // scroll to the bottom + $el[0].scrollTo(0, scrollHeight); + executionsTab.getters.executionListItems().should('have.length', 18); + executionsTab.getters.successfulExecutionListItems().should('have.length', 18); + executionsTab.getters.failedExecutionListItems().should('have.length', 0); + }); + + cy.getByTestId('executions-filter-button').click(); + cy.getByTestId('executions-filter-reset-button').should('be.visible').click(); + executionsTab.getters.executionListItems().eq(11).should('be.visible'); + }); }); const createMockExecutions = () => { diff --git a/cypress/e2e/233-AI-switch-to-logs-on-error.cy.ts b/cypress/e2e/233-AI-switch-to-logs-on-error.cy.ts new file mode 100644 index 0000000000..4c733df90d --- /dev/null +++ b/cypress/e2e/233-AI-switch-to-logs-on-error.cy.ts @@ -0,0 +1,279 @@ +import type { ExecutionError } from 'n8n-workflow/src'; +import { NDV, WorkflowPage as WorkflowPageClass } from '../pages'; +import { + addLanguageModelNodeToParent, + addMemoryNodeToParent, + addNodeToCanvas, + addToolNodeToParent, + navigateToNewWorkflowPage, + openNode, +} from '../composables/workflow'; +import { + AGENT_NODE_NAME, + AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, + AI_MEMORY_POSTGRES_NODE_NAME, + AI_TOOL_CALCULATOR_NODE_NAME, + CHAT_TRIGGER_NODE_DISPLAY_NAME, + MANUAL_CHAT_TRIGGER_NODE_NAME, + MANUAL_TRIGGER_NODE_DISPLAY_NAME, + MANUAL_TRIGGER_NODE_NAME, +} from '../constants'; +import { + clickCreateNewCredential, + clickExecuteNode, + clickGetBackToCanvas, +} from '../composables/ndv'; +import { setCredentialValues } from '../composables/modals/credential-modal'; +import { + closeManualChatModal, + getManualChatMessages, + getManualChatModalLogs, + getManualChatModalLogsEntries, + sendManualChatMessage, +} from '../composables/modals/chat-modal'; +import { createMockNodeExecutionData, getVisibleSelect, runMockWorkflowExecution } from '../utils'; + +const ndv = new NDV(); +const WorkflowPage = new WorkflowPageClass(); + +function createRunDataWithError(inputMessage: string) { + return [ + createMockNodeExecutionData(MANUAL_CHAT_TRIGGER_NODE_NAME, { + jsonData: { + main: { input: inputMessage }, + }, + }), + createMockNodeExecutionData(AI_MEMORY_POSTGRES_NODE_NAME, { + jsonData: { + ai_memory: { + json: { + action: 'loadMemoryVariables', + values: { + input: inputMessage, + system_message: 'You are a helpful assistant', + formatting_instructions: + 'IMPORTANT: Always call `format_final_response` to format your final response!', + }, + }, + }, + }, + inputOverride: { + ai_memory: [ + [ + { + json: { + action: 'loadMemoryVariables', + values: { + input: inputMessage, + system_message: 'You are a helpful assistant', + formatting_instructions: + 'IMPORTANT: Always call `format_final_response` to format your final response!', + }, + }, + }, + ], + ], + }, + error: { + message: 'Internal error', + timestamp: 1722591723244, + name: 'NodeOperationError', + description: 'Internal error', + context: {}, + cause: { + name: 'error', + severity: 'FATAL', + code: '3D000', + file: 'postinit.c', + line: '885', + routine: 'InitPostgres', + } as unknown as Error, + } as ExecutionError, + }), + createMockNodeExecutionData(AGENT_NODE_NAME, { + executionStatus: 'error', + error: { + level: 'error', + tags: { + packageName: 'workflow', + }, + context: {}, + functionality: 'configuration-node', + name: 'NodeOperationError', + timestamp: 1722591723244, + node: { + parameters: { + notice: '', + sessionIdType: 'fromInput', + tableName: 'n8n_chat_histories', + }, + id: '6b9141da-0135-4e9d-94d1-2d658cbf48b5', + name: 'Postgres Chat Memory', + type: '@n8n/n8n-nodes-langchain.memoryPostgresChat', + typeVersion: 1, + position: [1140, 500], + credentials: { + postgres: { + id: 'RkyZetVpGsSfEAhQ', + name: 'Postgres account', + }, + }, + }, + messages: ['database "chat11" does not exist'], + description: 'Internal error', + message: 'Internal error', + } as unknown as ExecutionError, + metadata: { + subRun: [ + { + node: 'Postgres Chat Memory', + runIndex: 0, + }, + ], + }, + }), + ]; +} + +function setupTestWorkflow(chatTrigger: boolean = false) { + // Setup test workflow with AI Agent, Postgres Memory Node (source of error), Calculator Tool, and OpenAI Chat Model + if (chatTrigger) { + addNodeToCanvas(MANUAL_CHAT_TRIGGER_NODE_NAME, true); + } else { + addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME, true); + } + + addNodeToCanvas(AGENT_NODE_NAME, true); + + if (!chatTrigger) { + // Remove chat trigger + WorkflowPage.getters + .canvasNodeByName(CHAT_TRIGGER_NODE_DISPLAY_NAME) + .find('[data-test-id="delete-node-button"]') + .click({ force: true }); + + // Set manual trigger to output standard pinned data + openNode(MANUAL_TRIGGER_NODE_DISPLAY_NAME); + ndv.actions.editPinnedData(); + ndv.actions.savePinnedData(); + ndv.actions.close(); + } + + // Calculator is added just to make OpenAI Chat Model work (tools can not be empty with OpenAI model) + addToolNodeToParent(AI_TOOL_CALCULATOR_NODE_NAME, AGENT_NODE_NAME); + clickGetBackToCanvas(); + + addMemoryNodeToParent(AI_MEMORY_POSTGRES_NODE_NAME, AGENT_NODE_NAME); + + clickCreateNewCredential(); + setCredentialValues({ + password: 'testtesttest', + }); + + ndv.getters.parameterInput('sessionIdType').click(); + getVisibleSelect().contains('Define below').click(); + ndv.getters.parameterInput('sessionKey').type('asdasd'); + + clickGetBackToCanvas(); + + addLanguageModelNodeToParent( + AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, + AGENT_NODE_NAME, + true, + ); + + clickCreateNewCredential(); + setCredentialValues({ + apiKey: 'sk_test_123', + }); + clickGetBackToCanvas(); + + WorkflowPage.actions.zoomToFit(); +} + +function checkMessages(inputMessage: string, outputMessage: string) { + const messages = getManualChatMessages(); + messages.should('have.length', 2); + messages.should('contain', inputMessage); + messages.should('contain', outputMessage); + + getManualChatModalLogs().should('exist'); + getManualChatModalLogsEntries() + .should('have.length', 1) + .should('contain', AI_MEMORY_POSTGRES_NODE_NAME); +} + +describe("AI-233 Make root node's logs pane active in case of an error in sub-nodes", () => { + beforeEach(() => { + navigateToNewWorkflowPage(); + }); + + it('should open logs tab by default when there was an error', () => { + setupTestWorkflow(true); + + openNode(AGENT_NODE_NAME); + + const inputMessage = 'Test the code tool'; + + clickExecuteNode(); + runMockWorkflowExecution({ + trigger: () => sendManualChatMessage(inputMessage), + runData: createRunDataWithError(inputMessage), + lastNodeExecuted: AGENT_NODE_NAME, + }); + + checkMessages(inputMessage, '[ERROR: Internal error]'); + closeManualChatModal(); + + // Open the AI Agent node to see the logs + openNode(AGENT_NODE_NAME); + + // Finally check that logs pane is opened by default + ndv.getters.outputDataContainer().should('be.visible'); + + ndv.getters.aiOutputModeToggle().should('be.visible'); + ndv.getters + .aiOutputModeToggle() + .find('[role="radio"]') + .should('have.length', 2) + .eq(1) + .should('have.attr', 'aria-checked', 'true'); + + ndv.getters + .outputPanel() + .findChildByTestId('node-error-message') + .should('be.visible') + .should('contain', 'Error in sub-node'); + }); + + it('should switch to logs tab on error, when NDV is already opened', () => { + setupTestWorkflow(false); + + openNode(AGENT_NODE_NAME); + + const inputMessage = 'Test the code tool'; + + runMockWorkflowExecution({ + trigger: () => clickExecuteNode(), + runData: createRunDataWithError(inputMessage), + lastNodeExecuted: AGENT_NODE_NAME, + }); + + // Check that logs pane is opened by default + ndv.getters.outputDataContainer().should('be.visible'); + + ndv.getters.aiOutputModeToggle().should('be.visible'); + ndv.getters + .aiOutputModeToggle() + .find('[role="radio"]') + .should('have.length', 2) + .eq(1) + .should('have.attr', 'aria-checked', 'true'); + + ndv.getters + .outputPanel() + .findChildByTestId('node-error-message') + .should('be.visible') + .should('contain', 'Error in sub-node'); + }); +}); diff --git a/cypress/e2e/30-langchain.cy.ts b/cypress/e2e/30-langchain.cy.ts index c1409a34f3..c6d0f4ab4d 100644 --- a/cypress/e2e/30-langchain.cy.ts +++ b/cypress/e2e/30-langchain.cy.ts @@ -10,7 +10,9 @@ import { disableNode, getExecuteWorkflowButton, navigateToNewWorkflowPage, + getNodes, openNode, + getConnectionBySourceAndTarget, } from '../composables/workflow'; import { clickCreateNewCredential, @@ -41,6 +43,7 @@ import { AI_TOOL_WIKIPEDIA_NODE_NAME, BASIC_LLM_CHAIN_NODE_NAME, EDIT_FIELDS_SET_NODE_NAME, + CHAT_TRIGGER_NODE_DISPLAY_NAME, } from './../constants'; describe('Langchain Integration', () => { @@ -331,4 +334,27 @@ describe('Langchain Integration', () => { closeManualChatModal(); }); + + it('should auto-add chat trigger and basic LLM chain when adding LLM node', () => { + addNodeToCanvas(AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, true); + + getConnectionBySourceAndTarget( + CHAT_TRIGGER_NODE_DISPLAY_NAME, + BASIC_LLM_CHAIN_NODE_NAME, + ).should('exist'); + + getConnectionBySourceAndTarget( + AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, + BASIC_LLM_CHAIN_NODE_NAME, + ).should('exist'); + getNodes().should('have.length', 3); + }); + + it('should not auto-add nodes if AI nodes are already present', () => { + addNodeToCanvas(AGENT_NODE_NAME, true); + + addNodeToCanvas(AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, true); + getConnectionBySourceAndTarget(CHAT_TRIGGER_NODE_DISPLAY_NAME, AGENT_NODE_NAME).should('exist'); + getNodes().should('have.length', 3); + }); }); diff --git a/cypress/e2e/33-settings-personal.cy.ts b/cypress/e2e/33-settings-personal.cy.ts index 73dc7476b8..e3cc572c9e 100644 --- a/cypress/e2e/33-settings-personal.cy.ts +++ b/cypress/e2e/33-settings-personal.cy.ts @@ -35,13 +35,14 @@ describe('Personal Settings', () => { successToast().find('.el-notification__closeBtn').click(); }); }); + // eslint-disable-next-line n8n-local-rules/no-skipped-tests it('not allow malicious values for personal data', () => { cy.visit('/settings/personal'); INVALID_NAMES.forEach((name) => { cy.getByTestId('personal-data-form').find('input[name="firstName"]').clear().type(name); cy.getByTestId('personal-data-form').find('input[name="lastName"]').clear().type(name); cy.getByTestId('save-settings-button').click(); - errorToast().should('contain', 'Malicious firstName | Malicious lastName'); + errorToast().should('contain', 'Potentially malicious string | Potentially malicious string'); errorToast().find('.el-notification__closeBtn').click(); }); }); diff --git a/cypress/e2e/39-projects.cy.ts b/cypress/e2e/39-projects.cy.ts index 94a6384233..e2bf63df7d 100644 --- a/cypress/e2e/39-projects.cy.ts +++ b/cypress/e2e/39-projects.cy.ts @@ -1,9 +1,4 @@ -import { - INSTANCE_MEMBERS, - INSTANCE_OWNER, - MANUAL_TRIGGER_NODE_NAME, - NOTION_NODE_NAME, -} from '../constants'; +import { INSTANCE_MEMBERS, MANUAL_TRIGGER_NODE_NAME, NOTION_NODE_NAME } from '../constants'; import { WorkflowsPage, WorkflowPage, @@ -11,9 +6,10 @@ import { CredentialsPage, WorkflowExecutionsTab, NDV, + MainSidebar, } from '../pages'; import * as projects from '../composables/projects'; -import { getVisibleSelect } from '../utils'; +import { getVisibleDropdown, getVisibleModalOverlay, getVisibleSelect } from '../utils'; const workflowsPage = new WorkflowsPage(); const workflowPage = new WorkflowPage(); @@ -21,6 +17,7 @@ const credentialsPage = new CredentialsPage(); const credentialsModal = new CredentialsModal(); const executionsTab = new WorkflowExecutionsTab(); const ndv = new NDV(); +const mainSidebar = new MainSidebar(); describe('Projects', { disableAutoLogin: true }, () => { before(() => { @@ -237,10 +234,30 @@ describe('Projects', { disableAutoLogin: true }, () => { cy.signinAsMember(1); cy.visit(workflowsPage.url); - projects.getAddProjectButton().should('not.exist'); + cy.getByTestId('add-project-menu-item').should('not.exist'); projects.getMenuItems().should('not.exist'); }); + it('should not show viewer role if not licensed', () => { + cy.signinAsOwner(); + cy.visit(workflowsPage.url); + + projects.getMenuItems().first().click(); + projects.getProjectTabSettings().click(); + + cy.get( + `[data-test-id="user-list-item-${INSTANCE_MEMBERS[0].email}"] [data-test-id="projects-settings-user-role-select"]`, + ).click(); + + cy.get('.el-select-dropdown__item.is-disabled') + .should('contain.text', 'Viewer') + .get('span:contains("Upgrade")') + .filter(':visible') + .click(); + + getVisibleModalOverlay().should('contain.text', 'Upgrade to unlock additional roles'); + }); + describe('when starting from scratch', () => { beforeEach(() => { cy.resetDatabase(); @@ -257,7 +274,7 @@ describe('Projects', { disableAutoLogin: true }, () => { // Create a project and add a credential to it cy.intercept('POST', '/rest/projects').as('projectCreate'); - projects.getAddProjectButton().should('contain', 'Add project').should('be.visible').click(); + projects.getAddProjectButton().click(); cy.wait('@projectCreate'); projects.getMenuItems().should('have.length', 1); projects.getMenuItems().first().click(); @@ -418,7 +435,7 @@ describe('Projects', { disableAutoLogin: true }, () => { }); it('should move resources between projects', () => { - cy.signin(INSTANCE_OWNER); + cy.signinAsOwner(); cy.visit(workflowsPage.url); // Create a workflow and a credential in the Home project @@ -563,5 +580,80 @@ describe('Projects', { disableAutoLogin: true }, () => { projects.getProjectTabCredentials().click(); credentialsPage.getters.credentialCards().should('have.length', 2); }); + + it('should handle viewer role', () => { + cy.enableFeature('projectRole:viewer'); + cy.signinAsOwner(); + cy.visit(workflowsPage.url); + + projects.createProject('Development'); + projects.addProjectMember(INSTANCE_MEMBERS[0].email, 'Viewer'); + projects.getProjectSettingsSaveButton().click(); + + projects.getProjectTabWorkflows().click(); + workflowsPage.getters.newWorkflowButtonCard().click(); + projects.createWorkflow('Test_workflow_4_executions_view.json', 'WF with random error'); + executionsTab.actions.createManualExecutions(2); + executionsTab.actions.toggleNodeEnabled('Error'); + executionsTab.actions.createManualExecutions(2); + workflowPage.actions.saveWorkflowUsingKeyboardShortcut(); + + projects.getMenuItems().first().click(); + projects.getProjectTabCredentials().click(); + credentialsPage.getters.emptyListCreateCredentialButton().click(); + projects.createCredential('Notion API'); + + mainSidebar.actions.openUserMenu(); + cy.getByTestId('user-menu-item-logout').click(); + + cy.get('input[name="email"]').type(INSTANCE_MEMBERS[0].email); + cy.get('input[name="password"]').type(INSTANCE_MEMBERS[0].password); + cy.getByTestId('form-submit-button').click(); + + mainSidebar.getters.executions().click(); + cy.getByTestId('global-execution-list-item').first().find('td:last button').click(); + getVisibleDropdown() + .find('li') + .filter(':contains("Retry")') + .should('have.class', 'is-disabled'); + getVisibleDropdown() + .find('li') + .filter(':contains("Delete")') + .should('have.class', 'is-disabled'); + + projects.getMenuItems().first().click(); + cy.getByTestId('workflow-card-name').should('be.visible').first().click(); + workflowPage.getters.nodeViewRoot().should('be.visible'); + workflowPage.getters.executeWorkflowButton().should('not.exist'); + workflowPage.getters.nodeCreatorPlusButton().should('not.exist'); + workflowPage.getters.canvasNodes().should('have.length', 3).last().click(); + cy.get('body').type('{backspace}'); + workflowPage.getters.canvasNodes().should('have.length', 3).last().rightclick(); + getVisibleDropdown() + .find('li') + .should('be.visible') + .filter( + ':contains("Open"), :contains("Copy"), :contains("Select all"), :contains("Clear selection")', + ) + .should('not.have.class', 'is-disabled'); + cy.get('body').type('{esc}'); + + executionsTab.actions.switchToExecutionsTab(); + cy.getByTestId('retry-execution-button') + .should('be.visible') + .find('.is-disabled') + .should('exist'); + cy.get('button:contains("Debug")').should('be.disabled'); + cy.get('button[title="Retry execution"]').should('be.disabled'); + cy.get('button[title="Delete this execution"]').should('be.disabled'); + + projects.getMenuItems().first().click(); + projects.getProjectTabCredentials().click(); + credentialsPage.getters.credentialCards().filter(':contains("Notion")').click(); + cy.getByTestId('node-credentials-config-container') + .should('be.visible') + .find('input') + .should('not.have.length'); + }); }); }); diff --git a/cypress/e2e/44-routing.cy.ts b/cypress/e2e/44-routing.cy.ts new file mode 100644 index 0000000000..67a092235b --- /dev/null +++ b/cypress/e2e/44-routing.cy.ts @@ -0,0 +1,26 @@ +import { WorkflowsPage as WorkflowsPageClass } from '../pages/workflows'; +import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; +import { EDIT_FIELDS_SET_NODE_NAME } from '../constants'; +import { getSaveChangesModal } from '../composables/modals/save-changes-modal'; + +const WorkflowsPage = new WorkflowsPageClass(); +const WorkflowPage = new WorkflowPageClass(); + +describe('Workflows', () => { + beforeEach(() => { + cy.visit(WorkflowsPage.url); + }); + + it('should ask to save unsaved changes before leaving route', () => { + WorkflowsPage.getters.newWorkflowButtonCard().should('be.visible'); + WorkflowsPage.getters.newWorkflowButtonCard().click(); + + cy.createFixtureWorkflow('Test_workflow_1.json', 'Empty State Card Workflow'); + + WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); + + cy.getByTestId('project-home-menu-item').click(); + + getSaveChangesModal().should('be.visible'); + }); +}); diff --git a/cypress/e2e/45-ai-assistant.cy.ts b/cypress/e2e/45-ai-assistant.cy.ts new file mode 100644 index 0000000000..4431007df0 --- /dev/null +++ b/cypress/e2e/45-ai-assistant.cy.ts @@ -0,0 +1,247 @@ +import { NDV, WorkflowPage } from '../pages'; +import { AIAssistant } from '../pages/features/ai-assistant'; + +const wf = new WorkflowPage(); +const ndv = new NDV(); +const aiAssistant = new AIAssistant(); + +describe('AI Assistant::disabled', () => { + beforeEach(() => { + aiAssistant.actions.disableAssistant(); + wf.actions.visit(); + }); + + it('does not show assistant button if feature is disabled', () => { + aiAssistant.getters.askAssistantFloatingButton().should('not.exist'); + }); +}); + +describe('AI Assistant::enabled', () => { + beforeEach(() => { + aiAssistant.actions.enableAssistant(); + wf.actions.visit(); + }); + + after(() => { + aiAssistant.actions.disableAssistant(); + }); + + it('renders placeholder UI', () => { + aiAssistant.getters.askAssistantFloatingButton().should('be.visible'); + aiAssistant.getters.askAssistantFloatingButton().click(); + aiAssistant.getters.askAssistantChat().should('be.visible'); + aiAssistant.getters.placeholderMessage().should('be.visible'); + aiAssistant.getters.chatInputWrapper().should('not.exist'); + aiAssistant.getters.closeChatButton().should('be.visible'); + aiAssistant.getters.closeChatButton().click(); + aiAssistant.getters.askAssistantChat().should('not.exist'); + }); + + it('should resize assistant chat up', () => { + aiAssistant.getters.askAssistantFloatingButton().click(); + aiAssistant.getters.askAssistantSidebarResizer().should('be.visible'); + aiAssistant.getters.askAssistantChat().then((element) => { + const { width, left } = element[0].getBoundingClientRect(); + cy.drag(aiAssistant.getters.askAssistantSidebarResizer(), [left - 10, 0], { + abs: true, + clickToFinish: true, + }); + aiAssistant.getters.askAssistantChat().then((newElement) => { + const newWidth = newElement[0].getBoundingClientRect().width; + expect(newWidth).to.be.greaterThan(width); + }); + }); + }); + + it('should resize assistant chat down', () => { + aiAssistant.getters.askAssistantFloatingButton().click(); + aiAssistant.getters.askAssistantSidebarResizer().should('be.visible'); + aiAssistant.getters.askAssistantChat().then((element) => { + const { width, left } = element[0].getBoundingClientRect(); + cy.drag(aiAssistant.getters.askAssistantSidebarResizer(), [left + 10, 0], { + abs: true, + clickToFinish: true, + }); + aiAssistant.getters.askAssistantChat().then((newElement) => { + const newWidth = newElement[0].getBoundingClientRect().width; + expect(newWidth).to.be.lessThan(width); + }); + }); + }); + + it('should start chat session from node error view', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/simple_message_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Stop and Error'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.chatMessagesAll().should('have.length', 1); + aiAssistant.getters + .chatMessagesAll() + .eq(0) + .should('contain.text', 'Hey, this is an assistant message'); + aiAssistant.getters.nodeErrorViewAssistantButton().should('be.disabled'); + }); + + it('should render chat input correctly', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/simple_message_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Stop and Error'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + // Send button should be disabled when input is empty + aiAssistant.getters.sendMessageButton().should('be.disabled'); + aiAssistant.getters.chatInput().type('Yo '); + aiAssistant.getters.sendMessageButton().should('not.be.disabled'); + aiAssistant.getters.chatInput().then((element) => { + const { height } = element[0].getBoundingClientRect(); + // Shift + Enter should add a new line + aiAssistant.getters.chatInput().type('Hello{shift+enter}there'); + aiAssistant.getters.chatInput().then((newElement) => { + const newHeight = newElement[0].getBoundingClientRect().height; + // Chat input should grow as user adds new lines + expect(newHeight).to.be.greaterThan(height); + aiAssistant.getters.sendMessageButton().click(); + cy.wait('@chatRequest'); + // New lines should be rendered as
in the chat + aiAssistant.getters.chatMessagesUser().should('have.length', 1); + aiAssistant.getters.chatMessagesUser().eq(0).find('br').should('have.length', 1); + // Chat input should be cleared now + aiAssistant.getters.chatInput().should('have.value', ''); + }); + }); + }); + + it('should render and handle quick replies', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/quick_reply_message_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Stop and Error'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.quickReplies().should('have.length', 2); + aiAssistant.getters.quickReplies().eq(0).click(); + cy.wait('@chatRequest'); + aiAssistant.getters.chatMessagesUser().should('have.length', 1); + aiAssistant.getters.chatMessagesUser().eq(0).should('contain.text', "Sure, let's do it"); + }); + + it('should send message to assistant when node is executed', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/simple_message_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Edit Fields'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.chatMessagesAssistant().should('have.length', 1); + // Executing the same node should sende a new message to the assistant automatically + ndv.getters.nodeExecuteButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.chatMessagesAssistant().should('have.length', 2); + }); + + it('should warn before starting a new session', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/simple_message_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Edit Fields'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.closeChatButton().click(); + ndv.getters.backToCanvas().click(); + wf.actions.openNode('Stop and Error'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + // Since we already have an active session, a warning should be shown + aiAssistant.getters.newAssistantSessionModal().should('be.visible'); + aiAssistant.getters + .newAssistantSessionModal() + .find('button') + .contains('Start new session') + .click(); + cy.wait('@chatRequest'); + // New session should start with initial assistant message + aiAssistant.getters.chatMessagesAll().should('have.length', 1); + }); + + it('should apply code diff to code node', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/code_diff_suggestion_response.json', + }).as('chatRequest'); + cy.intercept('POST', '/rest/ai-assistant/chat/apply-suggestion', { + statusCode: 200, + fixture: 'aiAssistant/apply_code_diff_response.json', + }).as('applySuggestion'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Code'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click({ force: true }); + cy.wait('@chatRequest'); + // Should have two assistant messages + aiAssistant.getters.chatMessagesAll().should('have.length', 2); + aiAssistant.getters.codeDiffs().should('have.length', 1); + aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1); + aiAssistant.getters.applyCodeDiffButtons().first().click(); + cy.wait('@applySuggestion'); + aiAssistant.getters.applyCodeDiffButtons().should('have.length', 0); + aiAssistant.getters.undoReplaceCodeButtons().should('have.length', 1); + aiAssistant.getters.codeReplacedMessage().should('be.visible'); + ndv.getters + .parameterInput('jsCode') + .get('.cm-content') + .should('contain.text', 'item.json.myNewField = 1'); + // Clicking undo should revert the code back but not call the assistant + aiAssistant.getters.undoReplaceCodeButtons().first().click(); + aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1); + aiAssistant.getters.codeReplacedMessage().should('not.exist'); + cy.get('@applySuggestion.all').then((interceptions) => { + expect(interceptions).to.have.length(1); + }); + ndv.getters + .parameterInput('jsCode') + .get('.cm-content') + .should('contain.text', 'item.json.myNewField = 1aaa'); + // Replacing the code again should also not call the assistant + cy.get('@applySuggestion.all').then((interceptions) => { + expect(interceptions).to.have.length(1); + }); + aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1); + aiAssistant.getters.applyCodeDiffButtons().first().click(); + ndv.getters + .parameterInput('jsCode') + .get('.cm-content') + .should('contain.text', 'item.json.myNewField = 1'); + }); + + it('should end chat session when `end_session` event is received', () => { + cy.intercept('POST', '/rest/ai-assistant/chat', { + statusCode: 200, + fixture: 'aiAssistant/end_session_response.json', + }).as('chatRequest'); + cy.createFixtureWorkflow('aiAssistant/test_workflow.json'); + wf.actions.openNode('Stop and Error'); + ndv.getters.nodeExecuteButton().click(); + aiAssistant.getters.nodeErrorViewAssistantButton().click(); + cy.wait('@chatRequest'); + aiAssistant.getters.chatMessagesSystem().should('have.length', 1); + aiAssistant.getters.chatMessagesSystem().first().should('contain.text', 'session has ended'); + }); +}); diff --git a/cypress/fixtures/aiAssistant/apply_code_diff_response.json b/cypress/fixtures/aiAssistant/apply_code_diff_response.json new file mode 100644 index 0000000000..8d7ada0b40 --- /dev/null +++ b/cypress/fixtures/aiAssistant/apply_code_diff_response.json @@ -0,0 +1,8 @@ +{ + "data": { + "sessionId": "f9130bd7-c078-4862-a38a-369b27b0ff20-e96eb9f7-d581-4684-b6a9-fd3dfe9fe1fb-emTezIGat7bQsDdtIlbti", + "parameters": { + "jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1;\n}\n\nreturn $input.all();" + } + } +} diff --git a/cypress/fixtures/aiAssistant/code_diff_suggestion_response.json b/cypress/fixtures/aiAssistant/code_diff_suggestion_response.json new file mode 100644 index 0000000000..8ee5d647fd --- /dev/null +++ b/cypress/fixtures/aiAssistant/code_diff_suggestion_response.json @@ -0,0 +1,23 @@ +{ + "sessionId": "1", + "messages": [ + { + "role": "assistant", + "type": "message", + "text": "Hi there! Here is my top solution to fix the error in your **Code** node 👇" + }, + { + "type": "code-diff", + "description": "Fix the syntax error by changing '1asd' to a valid value. In this case, it seems like '1' was intended.", + "suggestionId": "1", + "codeDiff": "@@ -2,2 +2,2 @@\n item.json.myNewField = 1asd;\n+ item.json.myNewField = 1;\n", + "role": "assistant", + "quickReplies": [ + { + "text": "Give me another solution", + "type": "new-suggestion" + } + ] + } + ] +} diff --git a/cypress/fixtures/aiAssistant/end_session_response.json b/cypress/fixtures/aiAssistant/end_session_response.json new file mode 100644 index 0000000000..c53574d93a --- /dev/null +++ b/cypress/fixtures/aiAssistant/end_session_response.json @@ -0,0 +1,16 @@ +{ + "sessionId": "f9130bd7-c078-4862-a38a-369b27b0ff20-e96eb9f7-d581-4684-b6a9-fd3dfe9fe1fb-XCldJLlusGrEVku5I9cYT", + "messages": [ + { + "role": "assistant", + "type": "agent-suggestion", + "title": "Glad to Help", + "text": "I'm glad I could help. If you have any more questions or need further assistance with your n8n workflows, feel free to ask!" + }, + { + "role": "assistant", + "type": "event", + "eventName": "end-session" + } + ] +} diff --git a/cypress/fixtures/aiAssistant/quick_reply_message_response.json b/cypress/fixtures/aiAssistant/quick_reply_message_response.json new file mode 100644 index 0000000000..a3c1b958c4 --- /dev/null +++ b/cypress/fixtures/aiAssistant/quick_reply_message_response.json @@ -0,0 +1,20 @@ +{ + "sessionId": "1", + "messages": [ + { + "role": "assistant", + "type": "message", + "text": "Hey, this is an assistant message", + "quickReplies": [ + { + "text": "Sure, let's do it", + "type": "yes" + }, + { + "text": "Nah, doesn't sound good", + "type": "no" + } + ] + } + ] +} diff --git a/cypress/fixtures/aiAssistant/simple_message_response.json b/cypress/fixtures/aiAssistant/simple_message_response.json new file mode 100644 index 0000000000..11299b91f9 --- /dev/null +++ b/cypress/fixtures/aiAssistant/simple_message_response.json @@ -0,0 +1,10 @@ +{ + "sessionId": "1", + "messages": [ + { + "role": "assistant", + "type": "message", + "text": "Hey, this is an assistant message" + } + ] +} diff --git a/cypress/fixtures/aiAssistant/test_workflow.json b/cypress/fixtures/aiAssistant/test_workflow.json new file mode 100644 index 0000000000..da930ea489 --- /dev/null +++ b/cypress/fixtures/aiAssistant/test_workflow.json @@ -0,0 +1,88 @@ +{ + "nodes": [ + { + "parameters": {}, + "id": "ebfced75-2ce1-4c41-a971-6c3b83522c4d", + "name": "When clicking ‘Test workflow’", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [ + 360, + 220 + ] + }, + { + "parameters": { + "errorMessage": "This is an error message" + }, + "id": "f2e60459-401a-49d5-acfc-7b2b31cfdcf7", + "name": "Stop and Error", + "type": "n8n-nodes-base.stopAndError", + "typeVersion": 1, + "position": [ + 1020, + 220 + ] + }, + { + "parameters": { + "jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1aaa;\n}\n\nreturn $input.all();" + }, + "id": "b54d4db9-b257-41a8-862f-26d293115bad", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 840, + 320 + ] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "053ada73-f7db-4e6a-8cc8-85756cc6ca4e", + "name": "age", + "value": "={{ 32sad }}", + "type": "number" + } + ] + }, + "options": {} + }, + "id": "5fd89612-a871-4679-b7b0-d659e09c6a0e", + "name": "Edit Fields", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 600, + 100 + ] + } + ], + "connections": { + "When clicking ‘Test workflow’": { + "main": [ + [ + { + "node": "Stop and Error", + "type": "main", + "index": 0 + }, + { + "node": "Code", + "type": "main", + "index": 0 + }, + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {} +} diff --git a/cypress/package.json b/cypress/package.json index 7740b5483f..5084e0b10c 100644 --- a/cypress/package.json +++ b/cypress/package.json @@ -14,7 +14,7 @@ "start": "cd ..; pnpm start" }, "devDependencies": { - "@types/lodash": "^4.14.195", + "@types/lodash": "catalog:", "eslint-plugin-cypress": "^3.3.0", "n8n-workflow": "workspace:*" }, @@ -24,8 +24,8 @@ "cypress": "^13.11.0", "cypress-otp": "^1.0.3", "cypress-real-events": "^1.12.0", - "lodash": "4.17.21", - "nanoid": "3.3.6", - "start-server-and-test": "^2.0.3" + "lodash": "catalog:", + "nanoid": "catalog:", + "start-server-and-test": "^2.0.5" } } diff --git a/cypress/pages/features/ai-assistant.ts b/cypress/pages/features/ai-assistant.ts new file mode 100644 index 0000000000..abca07fbbe --- /dev/null +++ b/cypress/pages/features/ai-assistant.ts @@ -0,0 +1,49 @@ +import { overrideFeatureFlag } from '../../composables/featureFlags'; +import { BasePage } from '../base'; + +const AI_ASSISTANT_FEATURE = { + name: 'aiAssistant', + experimentName: '021_ai_debug_helper', + enabledFor: 'variant', + disabledFor: 'control', +}; + +export class AIAssistant extends BasePage { + url = '/workflows/new'; + + getters = { + askAssistantFloatingButton: () => cy.getByTestId('ask-assistant-floating-button'), + askAssistantSidebar: () => cy.getByTestId('ask-assistant-sidebar'), + askAssistantSidebarResizer: () => + this.getters.askAssistantSidebar().find('[class^=_resizer][data-dir=left]').first(), + askAssistantChat: () => cy.getByTestId('ask-assistant-chat'), + placeholderMessage: () => cy.getByTestId('placeholder-message'), + closeChatButton: () => cy.getByTestId('close-chat-button'), + chatInputWrapper: () => cy.getByTestId('chat-input-wrapper'), + chatInput: () => cy.getByTestId('chat-input'), + sendMessageButton: () => cy.getByTestId('send-message-button'), + chatMessagesAll: () => cy.get('[data-test-id^=chat-message]'), + chatMessagesAssistant: () => cy.getByTestId('chat-message-assistant'), + chatMessagesUser: () => cy.getByTestId('chat-message-user'), + chatMessagesSystem: () => cy.getByTestId('chat-message-system'), + quickReplies: () => cy.getByTestId('quick-replies').find('button'), + newAssistantSessionModal: () => cy.getByTestId('new-assistant-session-modal'), + codeDiffs: () => cy.getByTestId('code-diff-suggestion'), + applyCodeDiffButtons: () => cy.getByTestId('replace-code-button'), + undoReplaceCodeButtons: () => cy.getByTestId('undo-replace-button'), + codeReplacedMessage: () => cy.getByTestId('code-replaced-message'), + nodeErrorViewAssistantButton: () => + cy.getByTestId('node-error-view-ask-assistant-button').find('button').first(), + }; + + actions = { + enableAssistant(): void { + overrideFeatureFlag(AI_ASSISTANT_FEATURE.experimentName, AI_ASSISTANT_FEATURE.enabledFor); + cy.enableFeature(AI_ASSISTANT_FEATURE.name); + }, + disableAssistant(): void { + overrideFeatureFlag(AI_ASSISTANT_FEATURE.experimentName, AI_ASSISTANT_FEATURE.disabledFor); + cy.disableFeature(AI_ASSISTANT_FEATURE.name); + }, + }; +} diff --git a/cypress/pages/ndv.ts b/cypress/pages/ndv.ts index 018ec43a5d..8bd7ccf95f 100644 --- a/cypress/pages/ndv.ts +++ b/cypress/pages/ndv.ts @@ -24,6 +24,7 @@ export class NDV extends BasePage { editPinnedDataButton: () => cy.getByTestId('ndv-edit-pinned-data'), pinnedDataEditor: () => this.getters.outputPanel().find('.cm-editor .cm-scroller .cm-content'), runDataPaneHeader: () => cy.getByTestId('run-data-pane-header'), + aiOutputModeToggle: () => cy.getByTestId('ai-output-mode-select'), nodeOutputHint: () => cy.getByTestId('ndv-output-run-node-hint'), savePinnedDataButton: () => this.getters.runDataPaneHeader().find('button').filter(':visible').contains('Save'), @@ -137,6 +138,8 @@ export class NDV extends BasePage { cy.getByTestId(`fixed-collection-${paramName}`), schemaViewNode: () => cy.getByTestId('run-data-schema-node'), schemaViewNodeName: () => cy.getByTestId('run-data-schema-node-name'), + expressionExpanders: () => cy.getByTestId('expander'), + expressionModalOutput: () => cy.getByTestId('expression-modal-output'), }; actions = { @@ -174,7 +177,7 @@ export class NDV extends BasePage { this.getters.editPinnedDataButton().click(); this.getters.pinnedDataEditor().click(); - this.getters.pinnedDataEditor().type('{selectall}{backspace}').paste(JSON.stringify(data)); + this.getters.pinnedDataEditor().invoke('text', '').paste(JSON.stringify(data)); this.actions.savePinnedData(); }, @@ -204,9 +207,9 @@ export class NDV extends BasePage { const droppable = `[data-test-id="parameter-input-${parameterName}"]`; cy.draganddrop(draggable, droppable); }, - mapToParameter: (parameterName: string) => { + mapToParameter: (parameterName: string, position?: 'top' | 'center' | 'bottom') => { const droppable = `[data-test-id="parameter-input-${parameterName}"]`; - cy.draganddrop('', droppable); + cy.draganddrop('', droppable, { position }); }, switchInputMode: (type: 'Schema' | 'Table' | 'JSON' | 'Binary') => { this.getters.inputDisplayMode().find('label').contains(type).click({ force: true }); diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index a7fa994289..1f353ab7c5 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -175,7 +175,7 @@ Cypress.Commands.add('drag', (selector, pos, options) => { }); }); -Cypress.Commands.add('draganddrop', (draggableSelector, droppableSelector) => { +Cypress.Commands.add('draganddrop', (draggableSelector, droppableSelector, options) => { if (draggableSelector) { cy.get(draggableSelector).should('exist'); } @@ -197,7 +197,7 @@ Cypress.Commands.add('draganddrop', (draggableSelector, droppableSelector) => { cy.get(droppableSelector).realMouseMove(0, 0); cy.get(droppableSelector).realMouseMove(pageX, pageY); cy.get(droppableSelector).realHover(); - cy.get(droppableSelector).realMouseUp(); + cy.get(droppableSelector).realMouseUp({ position: options?.position ?? 'top' }); if (draggableSelector) { cy.get(draggableSelector).realMouseUp(); } diff --git a/cypress/support/e2e.ts b/cypress/support/e2e.ts index 3968a09b5b..4d5d7a7f9a 100644 --- a/cypress/support/e2e.ts +++ b/cypress/support/e2e.ts @@ -17,6 +17,8 @@ beforeEach(() => { cy.window().then((win): void => { win.localStorage.setItem('N8N_THEME', 'light'); + win.localStorage.setItem('N8N_AUTOCOMPLETE_ONBOARDED', 'true'); + win.localStorage.setItem('N8N_MAPPING_ONBOARDED', 'true'); }); cy.intercept('GET', '/rest/settings', (req) => { diff --git a/cypress/support/index.ts b/cypress/support/index.ts index 9819e7c3a1..7c1897b11f 100644 --- a/cypress/support/index.ts +++ b/cypress/support/index.ts @@ -12,6 +12,10 @@ interface SigninPayload { password: string; } +interface DragAndDropOptions { + position: 'top' | 'center' | 'bottom'; +} + declare global { namespace Cypress { interface SuiteConfigOverrides { @@ -56,7 +60,11 @@ declare global { target: [number, number], options?: { abs?: boolean; index?: number; realMouse?: boolean; clickToFinish?: boolean }, ): void; - draganddrop(draggableSelector: string, droppableSelector: string): void; + draganddrop( + draggableSelector: string, + droppableSelector: string, + options?: Partial, + ): void; push(type: string, data: unknown): void; shouldNotHaveConsoleErrors(): void; window(): Chainable< diff --git a/cypress/utils/popper.ts b/cypress/utils/popper.ts index 5743c70f3e..43ef2997cf 100644 --- a/cypress/utils/popper.ts +++ b/cypress/utils/popper.ts @@ -3,7 +3,7 @@ export function getPopper() { } export function getVisiblePopper() { - return getPopper().filter(':visible'); + return getPopper().filter('[aria-hidden="false"]'); } export function getVisibleSelect() { diff --git a/docker/images/n8n/README.md b/docker/images/n8n/README.md index d654596150..f7b45f9467 100644 --- a/docker/images/n8n/README.md +++ b/docker/images/n8n/README.md @@ -230,6 +230,4 @@ Before you upgrade to the latest version make sure to check here if there are an ## License -n8n is [fair-code](https://faircode.io) distributed under the [**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md). - -Additional information about the license can be found in the [docs](https://docs.n8n.io/reference/license/). +You can find the license information [here](https://github.com/n8n-io/n8n/blob/master/README.md#license) diff --git a/package.json b/package.json index 59bae47e99..545a7c851f 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,19 @@ { "name": "n8n-monorepo", - "version": "1.52.0", + "version": "1.56.0", "private": true, "engines": { - "node": ">=18.10", - "pnpm": ">=9.1" + "node": ">=20.15", + "pnpm": ">=9.5" }, - "packageManager": "pnpm@9.1.4", + "packageManager": "pnpm@9.6.0", "scripts": { "preinstall": "node scripts/block-npm-install.js", "build": "turbo run build", "build:backend": "turbo run build:backend", "build:frontend": "turbo run build:frontend", "build:nodes": "turbo run build:nodes", - "typecheck": "turbo --filter=!n8n typecheck", + "typecheck": "turbo typecheck", "dev": "turbo run dev --parallel --env-mode=loose --filter=!n8n-design-system --filter=!@n8n/chat", "dev:ai": "turbo run dev --parallel --env-mode=loose --filter=@n8n/nodes-langchain --filter=n8n --filter=n8n-core", "clean": "turbo run clean --parallel", @@ -40,7 +40,6 @@ "@n8n_io/eslint-config": "workspace:*", "@types/jest": "^29.5.3", "@types/supertest": "^6.0.2", - "@vitest/coverage-v8": "^1.6.0", "jest": "^29.6.2", "jest-environment-jsdom": "^29.6.2", "jest-expect-message": "^1.1.3", @@ -56,11 +55,7 @@ "tsc-alias": "^1.8.7", "tsc-watch": "^6.0.4", "turbo": "2.0.6", - "typescript": "*", - "vite": "^5.2.12", - "vitest": "^1.6.0", - "vitest-mock-extended": "^1.3.1", - "vue-tsc": "^2.0.19" + "typescript": "*" }, "pnpm": { "onlyBuiltDependencies": [ @@ -68,7 +63,6 @@ ], "overrides": { "@types/node": "^18.16.16", - "axios": "1.6.7", "chokidar": "3.5.2", "esbuild": "^0.20.2", "formidable": "3.5.1", diff --git a/packages/@n8n/benchmark/Dockerfile b/packages/@n8n/benchmark/Dockerfile new file mode 100644 index 0000000000..5fa1aeae93 --- /dev/null +++ b/packages/@n8n/benchmark/Dockerfile @@ -0,0 +1,62 @@ +# syntax=docker/dockerfile:1 +FROM node:20.16.0 AS base + +# Install required dependencies +RUN apt-get update && apt-get install -y gnupg2 curl + +# Add k6 GPG key and repository +RUN mkdir -p /etc/apt/keyrings && \ + curl -sS https://dl.k6.io/key.gpg | gpg --dearmor --yes -o /etc/apt/keyrings/k6.gpg && \ + chmod a+x /etc/apt/keyrings/k6.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/k6.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list + +# Update and install k6 +RUN apt-get update && \ + apt-get install -y k6 tini && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable + +# +# Builder +FROM base AS builder + +WORKDIR /app + +COPY --chown=node:node ./pnpm-lock.yaml /app/pnpm-lock.yaml +COPY --chown=node:node ./pnpm-workspace.yaml /app/pnpm-workspace.yaml +COPY --chown=node:node ./package.json /app/package.json +COPY --chown=node:node ./packages/@n8n/benchmark/package.json /app/packages/@n8n/benchmark/package.json +COPY --chown=node:node ./patches /app/patches +COPY --chown=node:node ./scripts /app/scripts + +RUN pnpm install --frozen-lockfile + +# TS config files +COPY --chown=node:node ./tsconfig.json /app/tsconfig.json +COPY --chown=node:node ./tsconfig.build.json /app/tsconfig.build.json +COPY --chown=node:node ./tsconfig.backend.json /app/tsconfig.backend.json +COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.json /app/packages/@n8n/benchmark/tsconfig.json +COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.build.json /app/packages/@n8n/benchmark/tsconfig.build.json + +# Source files +COPY --chown=node:node ./packages/@n8n/benchmark/src /app/packages/@n8n/benchmark/src +COPY --chown=node:node ./packages/@n8n/benchmark/bin /app/packages/@n8n/benchmark/bin +COPY --chown=node:node ./packages/@n8n/benchmark/scenarios /app/packages/@n8n/benchmark/scenarios + +WORKDIR /app/packages/@n8n/benchmark +RUN pnpm build + +# +# Runner +FROM base AS runner + +COPY --from=builder /app /app + +WORKDIR /app/packages/@n8n/benchmark +USER node + +ENTRYPOINT [ "/app/packages/@n8n/benchmark/bin/n8n-benchmark" ] diff --git a/packages/@n8n/benchmark/README.md b/packages/@n8n/benchmark/README.md new file mode 100644 index 0000000000..569bcf897f --- /dev/null +++ b/packages/@n8n/benchmark/README.md @@ -0,0 +1,55 @@ +# n8n benchmarking tool + +Tool for executing benchmarks against an n8n instance. + +## Running locally with Docker + +Build the Docker image: + +```sh +# Must be run in the repository root +# k6 doesn't have an arm64 build available for linux, we need to build against amd64 +docker build --platform linux/amd64 -t n8n-benchmark -f packages/@n8n/benchmark/Dockerfile . +``` + +Run the image + +```sh +docker run \ + -e N8N_USER_EMAIL=user@n8n.io \ + -e N8N_USER_PASSWORD=password \ + # For macos, n8n running outside docker + -e N8N_BASE_URL=http://host.docker.internal:5678 \ + n8n-benchmark +``` + +## Running locally without Docker + +Requirements: + +- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) +- Node.js v20 or higher + +```sh +pnpm build + +# Run tests against http://localhost:5678 with specified email and password +N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run + +# If you installed k6 using brew, you might have to specify it explicitly +K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run +``` + +## Configuration + +The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts) + +## Benchmark scenarios + +A benchmark scenario defines one or multiple steps to execute and measure. It consists of: + +- Manifest file which describes and configures the scenario +- Any test data that is imported before the scenario is run +- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime. + +Available scenarios are located in [`./scenarios`](./scenarios/). diff --git a/packages/@n8n/benchmark/bin/n8n-benchmark b/packages/@n8n/benchmark/bin/n8n-benchmark new file mode 100755 index 0000000000..c7f0996f09 --- /dev/null +++ b/packages/@n8n/benchmark/bin/n8n-benchmark @@ -0,0 +1,13 @@ +#!/usr/bin/env node + +// Check if version should be displayed +const versionFlags = ['-v', '-V', '--version']; +if (versionFlags.includes(process.argv.slice(-1)[0])) { + console.log(require('../package').version); + process.exit(0); +} + +(async () => { + const oclif = require('@oclif/core'); + await oclif.execute({ dir: __dirname }); +})(); diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json new file mode 100644 index 0000000000..3a7afb50a3 --- /dev/null +++ b/packages/@n8n/benchmark/package.json @@ -0,0 +1,48 @@ +{ + "name": "@n8n/n8n-benchmark", + "version": "1.0.0", + "description": "Cli for running benchmark tests for n8n", + "main": "dist/index", + "scripts": { + "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json", + "start": "./bin/n8n-benchmark", + "test": "echo \"Error: no test specified\" && exit 1", + "typecheck": "tsc --noEmit", + "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" + }, + "engines": { + "node": ">=20.10" + }, + "keywords": [ + "automate", + "automation", + "IaaS", + "iPaaS", + "n8n", + "workflow", + "benchmark", + "performance" + ], + "dependencies": { + "@oclif/core": "4.0.7", + "axios": "catalog:", + "convict": "6.2.4", + "dotenv": "8.6.0", + "zx": "^8.1.4" + }, + "devDependencies": { + "@types/convict": "^6.1.1", + "@types/k6": "^0.52.0", + "@types/node": "^20.14.8", + "tsc-alias": "^1.8.7", + "typescript": "^5.5.2" + }, + "bin": { + "n8n-benchmark": "./bin/n8n-benchmark" + }, + "oclif": { + "bin": "n8n-benchmark", + "commands": "./dist/commands", + "topicSeparator": " " + } +} diff --git a/packages/@n8n/benchmark/scenarios/scenario.schema.json b/packages/@n8n/benchmark/scenarios/scenario.schema.json new file mode 100644 index 0000000000..661fc054b6 --- /dev/null +++ b/packages/@n8n/benchmark/scenarios/scenario.schema.json @@ -0,0 +1,42 @@ +{ + "definitions": { + "ScenarioData": { + "type": "object", + "properties": { + "workflowFiles": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [], + "additionalProperties": false + } + }, + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "The JSON schema to validate this file" + }, + "name": { + "type": "string", + "description": "The name of the scenario" + }, + "description": { + "type": "string", + "description": "A longer description of the scenario" + }, + "scriptPath": { + "type": "string", + "description": "Relative path to the k6 test script" + }, + "scenarioData": { + "$ref": "#/definitions/ScenarioData", + "description": "Data to import before running the scenario" + } + }, + "required": ["name", "description", "scriptPath", "scenarioData"], + "additionalProperties": false +} diff --git a/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.json b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.json new file mode 100644 index 0000000000..cba1aa5832 --- /dev/null +++ b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.json @@ -0,0 +1,25 @@ +{ + "createdAt": "2024-08-06T12:19:51.268Z", + "updatedAt": "2024-08-06T12:20:45.000Z", + "name": "Single Webhook", + "active": true, + "nodes": [ + { + "parameters": { "path": "single-webhook", "options": {} }, + "id": "7587ab0e-cc15-424f-83c0-c887a0eb97fb", + "name": "Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [760, 400], + "webhookId": "fa563fc2-c73f-4631-99a1-39c16f1f858f" + } + ], + "connections": {}, + "settings": { "executionOrder": "v1" }, + "staticData": null, + "meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} }, + "pinData": {}, + "versionId": "840a38a1-ba37-433d-9f20-de73f5131a2b", + "triggerCount": 1, + "tags": [] +} diff --git a/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.manifest.json b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.manifest.json new file mode 100644 index 0000000000..e9b4664a96 --- /dev/null +++ b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.manifest.json @@ -0,0 +1,7 @@ +{ + "$schema": "../scenario.schema.json", + "name": "SingleWebhook", + "description": "A single webhook trigger that responds with a 200 status code", + "scenarioData": { "workflowFiles": ["singleWebhook.json"] }, + "scriptPath": "singleWebhook.script.ts" +} diff --git a/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.script.ts b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.script.ts new file mode 100644 index 0000000000..72e2563cbe --- /dev/null +++ b/packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.script.ts @@ -0,0 +1,11 @@ +import http from 'k6/http'; +import { check } from 'k6'; + +const apiBaseUrl = __ENV.API_BASE_URL; + +export default function () { + const res = http.get(`${apiBaseUrl}/webhook/single-webhook`); + check(res, { + 'is status 200': (r) => r.status === 200, + }); +} diff --git a/packages/@n8n/benchmark/src/commands/list.ts b/packages/@n8n/benchmark/src/commands/list.ts new file mode 100644 index 0000000000..fcc60b1b81 --- /dev/null +++ b/packages/@n8n/benchmark/src/commands/list.ts @@ -0,0 +1,21 @@ +import { Command } from '@oclif/core'; +import { ScenarioLoader } from '@/scenario/scenarioLoader'; +import { loadConfig } from '@/config/config'; + +export default class ListCommand extends Command { + static description = 'List all available scenarios'; + + async run() { + const config = loadConfig(); + const scenarioLoader = new ScenarioLoader(); + + const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath')); + + console.log('Available test scenarios:'); + console.log(''); + + for (const scenario of allScenarios) { + console.log('\t', scenario.name, ':', scenario.description); + } + } +} diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts new file mode 100644 index 0000000000..d69b4a54d4 --- /dev/null +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -0,0 +1,39 @@ +import { Command, Flags } from '@oclif/core'; +import { loadConfig } from '@/config/config'; +import { ScenarioLoader } from '@/scenario/scenarioLoader'; +import { ScenarioRunner } from '@/testExecution/scenarioRunner'; +import { N8nApiClient } from '@/n8nApiClient/n8nApiClient'; +import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader'; +import { K6Executor } from '@/testExecution/k6Executor'; + +export default class RunCommand extends Command { + static description = 'Run all (default) or specified test scenarios'; + + // TODO: Add support for filtering scenarios + static flags = { + scenarios: Flags.string({ + char: 't', + description: 'Comma-separated list of test scenarios to run', + required: false, + }), + }; + + async run() { + const config = loadConfig(); + const scenarioLoader = new ScenarioLoader(); + + const scenarioRunner = new ScenarioRunner( + new N8nApiClient(config.get('n8n.baseUrl')), + new ScenarioDataFileLoader(), + new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')), + { + email: config.get('n8n.user.email'), + password: config.get('n8n.user.password'), + }, + ); + + const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath')); + + await scenarioRunner.runManyScenarios(allScenarios); + } +} diff --git a/packages/@n8n/benchmark/src/config/config.ts b/packages/@n8n/benchmark/src/config/config.ts new file mode 100644 index 0000000000..896ecc9296 --- /dev/null +++ b/packages/@n8n/benchmark/src/config/config.ts @@ -0,0 +1,50 @@ +import convict from 'convict'; +import dotenv from 'dotenv'; + +dotenv.config(); + +const configSchema = { + testScenariosPath: { + doc: 'The path to the scenarios', + format: String, + default: 'scenarios', + }, + n8n: { + baseUrl: { + doc: 'The base URL for the n8n instance', + format: String, + default: 'http://localhost:5678', + env: 'N8N_BASE_URL', + }, + user: { + email: { + doc: 'The email address of the n8n user', + format: String, + default: 'benchmark-user@n8n.io', + env: 'N8N_USER_EMAIL', + }, + password: { + doc: 'The password of the n8n user', + format: String, + default: 'VerySecret!123', + env: 'N8N_USER_PASSWORD', + }, + }, + }, + k6ExecutablePath: { + doc: 'The path to the k6 binary', + format: String, + default: 'k6', + env: 'K6_PATH', + }, +}; + +export type Config = ReturnType; + +export function loadConfig() { + const config = convict(configSchema); + + config.validate({ allowed: 'strict' }); + + return config; +} diff --git a/packages/@n8n/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts b/packages/@n8n/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts new file mode 100644 index 0000000000..93cd767347 --- /dev/null +++ b/packages/@n8n/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts @@ -0,0 +1,67 @@ +import { strict as assert } from 'node:assert'; +import { N8nApiClient } from './n8nApiClient'; +import { AxiosRequestConfig } from 'axios'; + +export class AuthenticatedN8nApiClient extends N8nApiClient { + constructor( + apiBaseUrl: string, + private readonly authCookie: string, + ) { + super(apiBaseUrl); + } + + static async createUsingUsernameAndPassword( + apiClient: N8nApiClient, + loginDetails: { + email: string; + password: string; + }, + ) { + const response = await apiClient.restApiRequest('/login', { + method: 'POST', + data: loginDetails, + }); + + const cookieHeader = response.headers['set-cookie']; + const authCookie = Array.isArray(cookieHeader) ? cookieHeader.join('; ') : cookieHeader; + assert(authCookie); + + return new AuthenticatedN8nApiClient(apiClient.apiBaseUrl, authCookie); + } + + async get(endpoint: string) { + return await this.authenticatedRequest(endpoint, { + method: 'GET', + }); + } + + async post(endpoint: string, data: unknown) { + return await this.authenticatedRequest(endpoint, { + method: 'POST', + data, + }); + } + + async patch(endpoint: string, data: unknown) { + return await this.authenticatedRequest(endpoint, { + method: 'PATCH', + data, + }); + } + + async delete(endpoint: string) { + return await this.authenticatedRequest(endpoint, { + method: 'DELETE', + }); + } + + protected async authenticatedRequest(endpoint: string, init: Omit) { + return await this.restApiRequest(endpoint, { + ...init, + headers: { + ...init.headers, + cookie: this.authCookie, + }, + }); + } +} diff --git a/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.ts b/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.ts new file mode 100644 index 0000000000..86ca52aff8 --- /dev/null +++ b/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.ts @@ -0,0 +1,78 @@ +import axios, { AxiosError, AxiosRequestConfig } from 'axios'; + +export class N8nApiClient { + constructor(public readonly apiBaseUrl: string) {} + + async waitForInstanceToBecomeOnline(): Promise { + const HEALTH_ENDPOINT = 'healthz'; + const START_TIME = Date.now(); + const INTERVAL_MS = 1000; + const TIMEOUT_MS = 60_000; + + while (Date.now() - START_TIME < TIMEOUT_MS) { + try { + const response = await axios.request({ + url: `${this.apiBaseUrl}/${HEALTH_ENDPOINT}`, + method: 'GET', + }); + + if (response.status === 200 && response.data.status === 'ok') { + return; + } + } catch {} + + console.log(`n8n instance not online yet, retrying in ${INTERVAL_MS / 1000} seconds...`); + await this.delay(INTERVAL_MS); + } + + throw new Error(`n8n instance did not come online within ${TIMEOUT_MS / 1000} seconds`); + } + + async setupOwnerIfNeeded(loginDetails: { email: string; password: string }) { + const response = await this.restApiRequest<{ message: string }>('/owner/setup', { + method: 'POST', + data: { + email: loginDetails.email, + password: loginDetails.password, + firstName: 'Test', + lastName: 'User', + }, + // Don't throw on non-2xx responses + validateStatus: () => true, + }); + + const responsePayload = response.data; + + if (response.status === 200) { + console.log('Owner setup successful'); + } else if (response.status === 400) { + if (responsePayload.message === 'Instance owner already setup') + console.log('Owner already set up'); + } else { + throw new Error( + `Owner setup failed with status ${response.status}: ${responsePayload.message}`, + ); + } + } + + async restApiRequest(endpoint: string, init: Omit) { + try { + return await axios.request({ + ...init, + url: this.getRestEndpointUrl(endpoint), + }); + } catch (e) { + const error = e as AxiosError; + console.error(`[ERROR] Request failed ${init.method} ${endpoint}`, error?.response?.data); + throw error; + } + } + + protected getRestEndpointUrl(endpoint: string) { + return `${this.apiBaseUrl}/rest${endpoint}`; + } + + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.types.ts b/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.types.ts new file mode 100644 index 0000000000..ff6aa6930b --- /dev/null +++ b/packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.types.ts @@ -0,0 +1,8 @@ +/** + * n8n workflow. This is a simplified version of the actual workflow object. + */ +export type Workflow = { + id: string; + name: string; + tags?: string[]; +}; diff --git a/packages/@n8n/benchmark/src/n8nApiClient/workflowsApiClient.ts b/packages/@n8n/benchmark/src/n8nApiClient/workflowsApiClient.ts new file mode 100644 index 0000000000..18f2ecbcda --- /dev/null +++ b/packages/@n8n/benchmark/src/n8nApiClient/workflowsApiClient.ts @@ -0,0 +1,31 @@ +import { Workflow } from '@/n8nApiClient/n8nApiClient.types'; +import { AuthenticatedN8nApiClient } from './authenticatedN8nApiClient'; + +export class WorkflowApiClient { + constructor(private readonly apiClient: AuthenticatedN8nApiClient) {} + + async getAllWorkflows(): Promise { + const response = await this.apiClient.get<{ count: number; data: Workflow[] }>('/workflows'); + + return response.data.data; + } + + async createWorkflow(workflow: unknown): Promise { + const response = await this.apiClient.post<{ data: Workflow }>('/workflows', workflow); + + return response.data.data; + } + + async activateWorkflow(workflow: Workflow): Promise { + const response = await this.apiClient.patch<{ data: Workflow }>(`/workflows/${workflow.id}`, { + ...workflow, + active: true, + }); + + return response.data.data; + } + + async deleteWorkflow(workflowId: Workflow['id']): Promise { + await this.apiClient.delete(`/workflows/${workflowId}`); + } +} diff --git a/packages/@n8n/benchmark/src/scenario/scenarioDataLoader.ts b/packages/@n8n/benchmark/src/scenario/scenarioDataLoader.ts new file mode 100644 index 0000000000..43638a2e00 --- /dev/null +++ b/packages/@n8n/benchmark/src/scenario/scenarioDataLoader.ts @@ -0,0 +1,35 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { Scenario } from '@/types/scenario'; +import { Workflow } from '@/n8nApiClient/n8nApiClient.types'; + +/** + * Loads scenario data files from FS + */ +export class ScenarioDataFileLoader { + async loadDataForScenario(scenario: Scenario): Promise<{ + workflows: Workflow[]; + }> { + const workflows = await Promise.all( + scenario.scenarioData.workflowFiles?.map((workflowFilePath) => + this.loadSingleWorkflowFromFile(path.join(scenario.scenarioDirPath, workflowFilePath)), + ) ?? [], + ); + + return { + workflows, + }; + } + + private loadSingleWorkflowFromFile(workflowFilePath: string): Workflow { + const fileContent = fs.readFileSync(workflowFilePath, 'utf8'); + + try { + return JSON.parse(fileContent); + } catch (error) { + throw new Error( + `Failed to parse workflow file ${workflowFilePath}: ${error instanceof Error ? error.message : error}`, + ); + } + } +} diff --git a/packages/@n8n/benchmark/src/scenario/scenarioLoader.ts b/packages/@n8n/benchmark/src/scenario/scenarioLoader.ts new file mode 100644 index 0000000000..475ce495ac --- /dev/null +++ b/packages/@n8n/benchmark/src/scenario/scenarioLoader.ts @@ -0,0 +1,67 @@ +import * as fs from 'node:fs'; +import * as path from 'path'; +import { createHash } from 'node:crypto'; +import type { Scenario, ScenarioManifest } from '@/types/scenario'; + +export class ScenarioLoader { + /** + * Loads all scenarios from the given path + */ + loadAll(pathToScenarios: string): Scenario[] { + pathToScenarios = path.resolve(pathToScenarios); + const scenarioFolders = fs + .readdirSync(pathToScenarios, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory()) + .map((dirent) => dirent.name); + + const scenarios: Scenario[] = []; + + for (const folder of scenarioFolders) { + const scenarioPath = path.join(pathToScenarios, folder); + const manifestFileName = `${folder}.manifest.json`; + const scenarioManifestPath = path.join(pathToScenarios, folder, manifestFileName); + if (!fs.existsSync(scenarioManifestPath)) { + console.warn(`Scenario at ${scenarioPath} is missing the ${manifestFileName} file`); + continue; + } + + // Load the scenario manifest file + const [scenario, validationErrors] = + this.loadAndValidateScenarioManifest(scenarioManifestPath); + if (validationErrors) { + console.warn( + `Scenario at ${scenarioPath} has the following validation errors: ${validationErrors.join(', ')}`, + ); + continue; + } + + scenarios.push({ + ...scenario, + id: this.formScenarioId(scenarioPath), + scenarioDirPath: scenarioPath, + }); + } + + return scenarios; + } + + private loadAndValidateScenarioManifest( + scenarioManifestPath: string, + ): [ScenarioManifest, null] | [null, string[]] { + const scenario = JSON.parse(fs.readFileSync(scenarioManifestPath, 'utf8')); + const validationErrors: string[] = []; + + if (!scenario.name) { + validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a name`); + } + if (!scenario.description) { + validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a description`); + } + + return validationErrors.length === 0 ? [scenario, null] : [null, validationErrors]; + } + + private formScenarioId(scenarioPath: string): string { + return createHash('sha256').update(scenarioPath).digest('hex'); + } +} diff --git a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts new file mode 100644 index 0000000000..903d06ca74 --- /dev/null +++ b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts @@ -0,0 +1,28 @@ +import { $ } from 'zx'; +import { Scenario } from '@/types/scenario'; + +/** + * Executes test scenarios using k6 + */ +export class K6Executor { + constructor( + private readonly k6ExecutablePath: string, + private readonly n8nApiBaseUrl: string, + ) {} + + async executeTestScenario(scenario: Scenario) { + // For 1 min with 5 virtual users + const stage = '1m:5'; + + const processPromise = $({ + cwd: scenario.scenarioDirPath, + env: { + API_BASE_URL: this.n8nApiBaseUrl, + }, + })`${this.k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`; + + for await (const chunk of processPromise.stdout) { + console.log(chunk.toString()); + } + } +} diff --git a/packages/@n8n/benchmark/src/testExecution/scenarioDataImporter.ts b/packages/@n8n/benchmark/src/testExecution/scenarioDataImporter.ts new file mode 100644 index 0000000000..1c1ec3777e --- /dev/null +++ b/packages/@n8n/benchmark/src/testExecution/scenarioDataImporter.ts @@ -0,0 +1,56 @@ +import { AuthenticatedN8nApiClient } from '@/n8nApiClient/authenticatedN8nApiClient'; +import { Workflow } from '@/n8nApiClient/n8nApiClient.types'; +import { WorkflowApiClient } from '@/n8nApiClient/workflowsApiClient'; + +/** + * Imports scenario data into an n8n instance + */ +export class ScenarioDataImporter { + private readonly workflowApiClient: WorkflowApiClient; + + constructor(n8nApiClient: AuthenticatedN8nApiClient) { + this.workflowApiClient = new WorkflowApiClient(n8nApiClient); + } + + async importTestScenarioData(workflows: Workflow[]) { + const existingWorkflows = await this.workflowApiClient.getAllWorkflows(); + + for (const workflow of workflows) { + await this.importWorkflow({ existingWorkflows, workflow }); + } + } + + /** + * Imports a single workflow into n8n removing any existing workflows with the same name + */ + private async importWorkflow(opts: { existingWorkflows: Workflow[]; workflow: Workflow }) { + const existingWorkflows = this.findExistingWorkflows(opts.existingWorkflows, opts.workflow); + if (existingWorkflows.length > 0) { + for (const toDelete of existingWorkflows) { + await this.workflowApiClient.deleteWorkflow(toDelete.id); + } + } + + const createdWorkflow = await this.workflowApiClient.createWorkflow({ + ...opts.workflow, + name: this.getBenchmarkWorkflowName(opts.workflow), + }); + + return await this.workflowApiClient.activateWorkflow(createdWorkflow); + } + + private findExistingWorkflows( + existingWorkflows: Workflow[], + workflowToImport: Workflow, + ): Workflow[] { + const benchmarkWorkflowName = this.getBenchmarkWorkflowName(workflowToImport); + + return existingWorkflows.filter( + (existingWorkflow) => existingWorkflow.name === benchmarkWorkflowName, + ); + } + + private getBenchmarkWorkflowName(workflow: Workflow) { + return `[BENCHMARK] ${workflow.name}`; + } +} diff --git a/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts b/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts new file mode 100644 index 0000000000..83e1077680 --- /dev/null +++ b/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts @@ -0,0 +1,50 @@ +import { Scenario } from '@/types/scenario'; +import { N8nApiClient } from '@/n8nApiClient/n8nApiClient'; +import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader'; +import { K6Executor } from './k6Executor'; +import { ScenarioDataImporter } from '@/testExecution/scenarioDataImporter'; +import { AuthenticatedN8nApiClient } from '@/n8nApiClient/authenticatedN8nApiClient'; + +/** + * Runs scenarios + */ +export class ScenarioRunner { + constructor( + private readonly n8nClient: N8nApiClient, + private readonly dataLoader: ScenarioDataFileLoader, + private readonly k6Executor: K6Executor, + private readonly ownerConfig: { + email: string; + password: string; + }, + ) {} + + async runManyScenarios(scenarios: Scenario[]) { + console.log(`Waiting for n8n ${this.n8nClient.apiBaseUrl} to become online`); + await this.n8nClient.waitForInstanceToBecomeOnline(); + + console.log('Setting up owner'); + await this.n8nClient.setupOwnerIfNeeded(this.ownerConfig); + + const authenticatedN8nClient = await AuthenticatedN8nApiClient.createUsingUsernameAndPassword( + this.n8nClient, + this.ownerConfig, + ); + const testDataImporter = new ScenarioDataImporter(authenticatedN8nClient); + + for (const scenario of scenarios) { + await this.runSingleTestScenario(testDataImporter, scenario); + } + } + + private async runSingleTestScenario(testDataImporter: ScenarioDataImporter, scenario: Scenario) { + console.log('Running scenario:', scenario.name); + + console.log('Loading and importing data'); + const testData = await this.dataLoader.loadDataForScenario(scenario); + await testDataImporter.importTestScenarioData(testData.workflows); + + console.log('Executing scenario script'); + await this.k6Executor.executeTestScenario(scenario); + } +} diff --git a/packages/@n8n/benchmark/src/types/scenario.ts b/packages/@n8n/benchmark/src/types/scenario.ts new file mode 100644 index 0000000000..19c52fd45b --- /dev/null +++ b/packages/@n8n/benchmark/src/types/scenario.ts @@ -0,0 +1,27 @@ +export type ScenarioData = { + /** Relative paths to the workflow files */ + workflowFiles?: string[]; +}; + +/** + * Configuration that defines the benchmark scenario + */ +export type ScenarioManifest = { + /** The name of the scenario */ + name: string; + /** A longer description of the scenario */ + description: string; + /** Relative path to the k6 script */ + scriptPath: string; + /** Data to import before running the scenario */ + scenarioData: ScenarioData; +}; + +/** + * Scenario with additional metadata + */ +export type Scenario = ScenarioManifest & { + id: string; + /** Path to the directory containing the scenario */ + scenarioDirPath: string; +}; diff --git a/packages/@n8n/benchmark/tsconfig.build.json b/packages/@n8n/benchmark/tsconfig.build.json new file mode 100644 index 0000000000..b91db37a4a --- /dev/null +++ b/packages/@n8n/benchmark/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": ["./tsconfig.json", "../../../tsconfig.build.json"], + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/build.tsbuildinfo" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/@n8n/benchmark/tsconfig.json b/packages/@n8n/benchmark/tsconfig.json new file mode 100644 index 0000000000..58a1b48f65 --- /dev/null +++ b/packages/@n8n/benchmark/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": ["../../../tsconfig.json", "../../../tsconfig.backend.json"], + "compilerOptions": { + "rootDir": ".", + "baseUrl": "src", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/@n8n/chat/README.md b/packages/@n8n/chat/README.md index 538d72ce0a..0ed53a5774 100644 --- a/packages/@n8n/chat/README.md +++ b/packages/@n8n/chat/README.md @@ -260,10 +260,5 @@ body, ``` ## License -n8n Chat is [fair-code](https://faircode.io) distributed under the -[**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md). -Proprietary licenses are available for enterprise customers. [Get in touch](mailto:license@n8n.io) - -Additional information about the license model can be found in the -[docs](https://docs.n8n.io/reference/license/). +You can find the license information [here](https://github.com/n8n-io/n8n/blob/master/README.md#license) diff --git a/packages/@n8n/chat/package.json b/packages/@n8n/chat/package.json index 25b9212b3e..6508081f45 100644 --- a/packages/@n8n/chat/package.json +++ b/packages/@n8n/chat/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/chat", - "version": "0.21.0", + "version": "0.24.0", "scripts": { "dev": "pnpm run storybook", "build": "pnpm build:vite && pnpm build:bundle", @@ -38,16 +38,19 @@ "@vueuse/core": "^10.11.0", "highlight.js": "^11.8.0", "markdown-it-link-attributes": "^4.0.1", - "uuid": "^8.3.2", - "vue": "^3.4.21", - "vue-markdown-render": "^2.1.1" + "uuid": "catalog:", + "vue": "catalog:frontend", + "vue-markdown-render": "catalog:frontend" }, "devDependencies": { "@iconify-json/mdi": "^1.1.54", "@n8n/storybook": "workspace:*", - "@types/markdown-it": "^12.2.3", + "@vitest/coverage-v8": "catalog:frontend", "unplugin-icons": "^0.19.0", - "vite-plugin-dts": "^3.9.1" + "vite": "catalog:frontend", + "vitest": "catalog:frontend", + "vite-plugin-dts": "^3.9.1", + "vue-tsc": "catalog:frontend" }, "files": [ "README.md", diff --git a/packages/@n8n/chat/src/components/Input.vue b/packages/@n8n/chat/src/components/Input.vue index 0415396082..af4b3343b7 100644 --- a/packages/@n8n/chat/src/components/Input.vue +++ b/packages/@n8n/chat/src/components/Input.vue @@ -247,6 +247,10 @@ function onOpenFileDialog() { justify-content: center; transition: color var(--chat--transition-duration) ease; + svg { + min-width: fit-content; + } + &:hover, &:focus { background: var( diff --git a/packages/@n8n/client-oauth2/package.json b/packages/@n8n/client-oauth2/package.json index 38ce65e8ee..7ec91b9839 100644 --- a/packages/@n8n/client-oauth2/package.json +++ b/packages/@n8n/client-oauth2/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/client-oauth2", - "version": "0.19.0", + "version": "0.20.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", @@ -20,6 +20,6 @@ "dist/**/*" ], "dependencies": { - "axios": "1.6.7" + "axios": "catalog:" } } diff --git a/packages/@n8n/config/.eslintrc.js b/packages/@n8n/config/.eslintrc.js index 032e99b09e..e5a8f3f0f9 100644 --- a/packages/@n8n/config/.eslintrc.js +++ b/packages/@n8n/config/.eslintrc.js @@ -7,4 +7,13 @@ module.exports = { extends: ['@n8n_io/eslint-config/node'], ...sharedOptions(__dirname), + + overrides: [ + { + files: ['**/*.config.ts'], + rules: { + 'n8n-local-rules/no-untyped-config-class-field': 'error', + }, + }, + ], }; diff --git a/packages/@n8n/config/package.json b/packages/@n8n/config/package.json index d992cc5aee..36d280a71f 100644 --- a/packages/@n8n/config/package.json +++ b/packages/@n8n/config/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/config", - "version": "1.2.0", + "version": "1.6.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", @@ -21,6 +21,6 @@ ], "dependencies": { "reflect-metadata": "0.2.2", - "typedi": "0.10.0" + "typedi": "catalog:" } } diff --git a/packages/@n8n/config/src/configs/cache.config.ts b/packages/@n8n/config/src/configs/cache.config.ts new file mode 100644 index 0000000000..fa1295d583 --- /dev/null +++ b/packages/@n8n/config/src/configs/cache.config.ts @@ -0,0 +1,36 @@ +import { Config, Env, Nested } from '../decorators'; + +@Config +class MemoryConfig { + /** Max size of memory cache in bytes */ + @Env('N8N_CACHE_MEMORY_MAX_SIZE') + maxSize: number = 3 * 1024 * 1024; // 3 MiB + + /** Time to live (in milliseconds) for data cached in memory. */ + @Env('N8N_CACHE_MEMORY_TTL') + ttl: number = 3600 * 1000; // 1 hour +} + +@Config +class RedisConfig { + /** Prefix for cache keys in Redis. */ + @Env('N8N_CACHE_REDIS_KEY_PREFIX') + prefix: string = 'redis'; + + /** Time to live (in milliseconds) for data cached in Redis. 0 for no TTL. */ + @Env('N8N_CACHE_REDIS_TTL') + ttl: number = 3600 * 1000; // 1 hour +} + +@Config +export class CacheConfig { + /** Backend to use for caching. */ + @Env('N8N_CACHE_BACKEND') + backend: 'memory' | 'redis' | 'auto' = 'auto'; + + @Nested + memory: MemoryConfig; + + @Nested + redis: RedisConfig; +} diff --git a/packages/@n8n/config/src/configs/credentials.ts b/packages/@n8n/config/src/configs/credentials.config.ts similarity index 77% rename from packages/@n8n/config/src/configs/credentials.ts rename to packages/@n8n/config/src/configs/credentials.config.ts index 9659061c05..f40e6b9f15 100644 --- a/packages/@n8n/config/src/configs/credentials.ts +++ b/packages/@n8n/config/src/configs/credentials.config.ts @@ -7,19 +7,19 @@ class CredentialsOverwrite { * Format: { CREDENTIAL_NAME: { PARAMETER: VALUE }} */ @Env('CREDENTIALS_OVERWRITE_DATA') - readonly data: string = '{}'; + data: string = '{}'; /** Internal API endpoint to fetch overwritten credential types from. */ @Env('CREDENTIALS_OVERWRITE_ENDPOINT') - readonly endpoint: string = ''; + endpoint: string = ''; } @Config export class CredentialsConfig { /** Default name for credentials */ @Env('CREDENTIALS_DEFAULT_NAME') - readonly defaultName: string = 'My credentials'; + defaultName: string = 'My credentials'; @Nested - readonly overwrite: CredentialsOverwrite; + overwrite: CredentialsOverwrite; } diff --git a/packages/@n8n/config/src/configs/database.ts b/packages/@n8n/config/src/configs/database.config.ts similarity index 70% rename from packages/@n8n/config/src/configs/database.ts rename to packages/@n8n/config/src/configs/database.config.ts index 384ecb1fb0..9da83958e6 100644 --- a/packages/@n8n/config/src/configs/database.ts +++ b/packages/@n8n/config/src/configs/database.config.ts @@ -4,19 +4,19 @@ import { Config, Env, Nested } from '../decorators'; class LoggingConfig { /** Whether database logging is enabled. */ @Env('DB_LOGGING_ENABLED') - readonly enabled: boolean = false; + enabled: boolean = false; /** * Database logging level. Requires `DB_LOGGING_MAX_EXECUTION_TIME` to be higher than `0`. */ @Env('DB_LOGGING_OPTIONS') - readonly options: 'query' | 'error' | 'schema' | 'warn' | 'info' | 'log' | 'all' = 'error'; + options: 'query' | 'error' | 'schema' | 'warn' | 'info' | 'log' | 'all' = 'error'; /** * Only queries that exceed this time (ms) will be logged. Set `0` to disable. */ @Env('DB_LOGGING_MAX_EXECUTION_TIME') - readonly maxQueryExecutionTime: number = 0; + maxQueryExecutionTime: number = 0; } @Config @@ -26,23 +26,23 @@ class PostgresSSLConfig { * If `DB_POSTGRESDB_SSL_CA`, `DB_POSTGRESDB_SSL_CERT`, or `DB_POSTGRESDB_SSL_KEY` are defined, `DB_POSTGRESDB_SSL_ENABLED` defaults to `true`. */ @Env('DB_POSTGRESDB_SSL_ENABLED') - readonly enabled: boolean = false; + enabled: boolean = false; /** SSL certificate authority */ @Env('DB_POSTGRESDB_SSL_CA') - readonly ca: string = ''; + ca: string = ''; /** SSL certificate */ @Env('DB_POSTGRESDB_SSL_CERT') - readonly cert: string = ''; + cert: string = ''; /** SSL key */ @Env('DB_POSTGRESDB_SSL_KEY') - readonly key: string = ''; + key: string = ''; /** If unauthorized SSL connections should be rejected */ @Env('DB_POSTGRESDB_SSL_REJECT_UNAUTHORIZED') - readonly rejectUnauthorized: boolean = true; + rejectUnauthorized: boolean = true; } @Config @@ -53,30 +53,30 @@ class PostgresConfig { /** Postgres database host */ @Env('DB_POSTGRESDB_HOST') - readonly host: string = 'localhost'; + host: string = 'localhost'; /** Postgres database password */ @Env('DB_POSTGRESDB_PASSWORD') - readonly password: string = ''; + password: string = ''; /** Postgres database port */ @Env('DB_POSTGRESDB_PORT') - readonly port: number = 5432; + port: number = 5432; /** Postgres database user */ @Env('DB_POSTGRESDB_USER') - readonly user: string = 'postgres'; + user: string = 'postgres'; /** Postgres database schema */ @Env('DB_POSTGRESDB_SCHEMA') - readonly schema: string = 'public'; + schema: string = 'public'; /** Postgres database pool size */ @Env('DB_POSTGRESDB_POOL_SIZE') - readonly poolSize = 2; + poolSize: number = 2; @Nested - readonly ssl: PostgresSSLConfig; + ssl: PostgresSSLConfig; } @Config @@ -87,36 +87,36 @@ class MysqlConfig { /** MySQL database host */ @Env('DB_MYSQLDB_HOST') - readonly host: string = 'localhost'; + host: string = 'localhost'; /** MySQL database password */ @Env('DB_MYSQLDB_PASSWORD') - readonly password: string = ''; + password: string = ''; /** MySQL database port */ @Env('DB_MYSQLDB_PORT') - readonly port: number = 3306; + port: number = 3306; /** MySQL database user */ @Env('DB_MYSQLDB_USER') - readonly user: string = 'root'; + user: string = 'root'; } @Config class SqliteConfig { /** SQLite database file name */ @Env('DB_SQLITE_DATABASE') - readonly database: string = 'database.sqlite'; + database: string = 'database.sqlite'; /** SQLite database pool size. Set to `0` to disable pooling. */ @Env('DB_SQLITE_POOL_SIZE') - readonly poolSize: number = 0; + poolSize: number = 0; /** * Enable SQLite WAL mode. */ @Env('DB_SQLITE_ENABLE_WAL') - readonly enableWAL: boolean = this.poolSize > 1; + enableWAL: boolean = this.poolSize > 1; /** * Run `VACUUM` on startup to rebuild the database, reducing file size and optimizing indexes. @@ -124,7 +124,7 @@ class SqliteConfig { * @warning Long-running blocking operation that will increase startup time. */ @Env('DB_SQLITE_VACUUM_ON_STARTUP') - readonly executeVacuumOnStartup: boolean = false; + executeVacuumOnStartup: boolean = false; } @Config @@ -135,17 +135,17 @@ export class DatabaseConfig { /** Prefix for table names */ @Env('DB_TABLE_PREFIX') - readonly tablePrefix: string = ''; + tablePrefix: string = ''; @Nested - readonly logging: LoggingConfig; + logging: LoggingConfig; @Nested - readonly postgresdb: PostgresConfig; + postgresdb: PostgresConfig; @Nested - readonly mysqldb: MysqlConfig; + mysqldb: MysqlConfig; @Nested - readonly sqlite: SqliteConfig; + sqlite: SqliteConfig; } diff --git a/packages/@n8n/config/src/configs/endpoints.config.ts b/packages/@n8n/config/src/configs/endpoints.config.ts new file mode 100644 index 0000000000..88efa01d26 --- /dev/null +++ b/packages/@n8n/config/src/configs/endpoints.config.ts @@ -0,0 +1,102 @@ +import { Config, Env, Nested } from '../decorators'; + +@Config +class PrometheusMetricsConfig { + /** Whether to enable the `/metrics` endpoint to expose Prometheus metrics. */ + @Env('N8N_METRICS') + enable: boolean = false; + + /** Prefix for Prometheus metric names. */ + @Env('N8N_METRICS_PREFIX') + prefix: string = 'n8n_'; + + /** Whether to expose system and Node.js metrics. See: https://www.npmjs.com/package/prom-client */ + @Env('N8N_METRICS_INCLUDE_DEFAULT_METRICS') + includeDefaultMetrics: boolean = true; + + /** Whether to include a label for workflow ID on workflow metrics. */ + @Env('N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL') + includeWorkflowIdLabel: boolean = false; + + /** Whether to include a label for node type on node metrics. */ + @Env('N8N_METRICS_INCLUDE_NODE_TYPE_LABEL') + includeNodeTypeLabel: boolean = false; + + /** Whether to include a label for credential type on credential metrics. */ + @Env('N8N_METRICS_INCLUDE_CREDENTIAL_TYPE_LABEL') + includeCredentialTypeLabel: boolean = false; + + /** Whether to expose metrics for API endpoints. See: https://www.npmjs.com/package/express-prom-bundle */ + @Env('N8N_METRICS_INCLUDE_API_ENDPOINTS') + includeApiEndpoints: boolean = false; + + /** Whether to include a label for the path of API endpoint calls. */ + @Env('N8N_METRICS_INCLUDE_API_PATH_LABEL') + includeApiPathLabel: boolean = false; + + /** Whether to include a label for the HTTP method of API endpoint calls. */ + @Env('N8N_METRICS_INCLUDE_API_METHOD_LABEL') + includeApiMethodLabel: boolean = false; + + /** Whether to include a label for the status code of API endpoint calls. */ + @Env('N8N_METRICS_INCLUDE_API_STATUS_CODE_LABEL') + includeApiStatusCodeLabel: boolean = false; + + /** Whether to include metrics for cache hits and misses. */ + @Env('N8N_METRICS_INCLUDE_CACHE_METRICS') + includeCacheMetrics: boolean = false; + + /** Whether to include metrics derived from n8n's internal events */ + @Env('N8N_METRICS_INCLUDE_MESSAGE_EVENT_BUS_METRICS') + includeMessageEventBusMetrics: boolean = false; +} + +@Config +export class EndpointsConfig { + /** Max payload size in MiB */ + @Env('N8N_PAYLOAD_SIZE_MAX') + payloadSizeMax: number = 16; + + @Nested + metrics: PrometheusMetricsConfig; + + /** Path segment for REST API endpoints. */ + @Env('N8N_ENDPOINT_REST') + rest: string = 'rest'; + + /** Path segment for form endpoints. */ + @Env('N8N_ENDPOINT_FORM') + form: string = 'form'; + + /** Path segment for test form endpoints. */ + @Env('N8N_ENDPOINT_FORM_TEST') + formTest: string = 'form-test'; + + /** Path segment for waiting form endpoints. */ + @Env('N8N_ENDPOINT_FORM_WAIT') + formWaiting: string = 'form-waiting'; + + /** Path segment for webhook endpoints. */ + @Env('N8N_ENDPOINT_WEBHOOK') + webhook: string = 'webhook'; + + /** Path segment for test webhook endpoints. */ + @Env('N8N_ENDPOINT_WEBHOOK_TEST') + webhookTest: string = 'webhook-test'; + + /** Path segment for waiting webhook endpoints. */ + @Env('N8N_ENDPOINT_WEBHOOK_WAIT') + webhookWaiting: string = 'webhook-waiting'; + + /** Whether to disable n8n's UI (frontend). */ + @Env('N8N_DISABLE_UI') + disableUi: boolean = false; + + /** Whether to disable production webhooks on the main process, when using webhook-specific processes. */ + @Env('N8N_DISABLE_PRODUCTION_MAIN_PROCESS') + disableProductionWebhooksOnMainProcess: boolean = false; + + /** Colon-delimited list of additional endpoints to not open the UI on. */ + @Env('N8N_ADDITIONAL_NON_UI_ROUTES') + additionalNonUIRoutes: string = ''; +} diff --git a/packages/@n8n/config/src/configs/event-bus.ts b/packages/@n8n/config/src/configs/event-bus.config.ts similarity index 69% rename from packages/@n8n/config/src/configs/event-bus.ts rename to packages/@n8n/config/src/configs/event-bus.config.ts index ed1226fa92..b4782555d5 100644 --- a/packages/@n8n/config/src/configs/event-bus.ts +++ b/packages/@n8n/config/src/configs/event-bus.config.ts @@ -2,30 +2,30 @@ import { Config, Env, Nested } from '../decorators'; @Config class LogWriterConfig { - /** Number of event log files to keep */ + /* of event log files to keep */ @Env('N8N_EVENTBUS_LOGWRITER_KEEPLOGCOUNT') - readonly keepLogCount: number = 3; + keepLogCount: number = 3; /** Max size (in KB) of an event log file before a new one is started */ @Env('N8N_EVENTBUS_LOGWRITER_MAXFILESIZEINKB') - readonly maxFileSizeInKB: number = 10240; // 10 MB + maxFileSizeInKB: number = 10240; // 10 MB /** Basename of event log file */ @Env('N8N_EVENTBUS_LOGWRITER_LOGBASENAME') - readonly logBaseName: string = 'n8nEventLog'; + logBaseName: string = 'n8nEventLog'; } @Config export class EventBusConfig { /** How often (in ms) to check for unsent event messages. Can in rare cases cause a message to be sent twice. `0` to disable */ @Env('N8N_EVENTBUS_CHECKUNSENTINTERVAL') - readonly checkUnsentInterval: number = 0; + checkUnsentInterval: number = 0; /** Endpoint to retrieve n8n version information from */ @Nested - readonly logWriter: LogWriterConfig; + logWriter: LogWriterConfig; /** Whether to recover execution details after a crash or only mark status executions as crashed. */ @Env('N8N_EVENTBUS_RECOVERY_MODE') - readonly crashRecoveryMode: 'simple' | 'extensive' = 'extensive'; + crashRecoveryMode: 'simple' | 'extensive' = 'extensive'; } diff --git a/packages/@n8n/config/src/configs/external-secrets.ts b/packages/@n8n/config/src/configs/external-secrets.config.ts similarity index 80% rename from packages/@n8n/config/src/configs/external-secrets.ts rename to packages/@n8n/config/src/configs/external-secrets.config.ts index a5310d675e..1195adf660 100644 --- a/packages/@n8n/config/src/configs/external-secrets.ts +++ b/packages/@n8n/config/src/configs/external-secrets.config.ts @@ -4,9 +4,9 @@ import { Config, Env } from '../decorators'; export class ExternalSecretsConfig { /** How often (in seconds) to check for secret updates */ @Env('N8N_EXTERNAL_SECRETS_UPDATE_INTERVAL') - readonly updateInterval: number = 300; + updateInterval: number = 300; /** Whether to prefer GET over LIST when fetching secrets from Hashicorp Vault */ @Env('N8N_EXTERNAL_SECRETS_PREFER_GET') - readonly preferGet: boolean = false; + preferGet: boolean = false; } diff --git a/packages/@n8n/config/src/configs/external-storage.ts b/packages/@n8n/config/src/configs/external-storage.config.ts similarity index 75% rename from packages/@n8n/config/src/configs/external-storage.ts rename to packages/@n8n/config/src/configs/external-storage.config.ts index c876e0ee34..6e5fbd64d8 100644 --- a/packages/@n8n/config/src/configs/external-storage.ts +++ b/packages/@n8n/config/src/configs/external-storage.config.ts @@ -4,39 +4,39 @@ import { Config, Env, Nested } from '../decorators'; class S3BucketConfig { /** Name of the n8n bucket in S3-compatible external storage */ @Env('N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME') - readonly name: string = ''; + name: string = ''; /** Region of the n8n bucket in S3-compatible external storage @example "us-east-1" */ @Env('N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION') - readonly region: string = ''; + region: string = ''; } @Config class S3CredentialsConfig { /** Access key in S3-compatible external storage */ @Env('N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY') - readonly accessKey: string = ''; + accessKey: string = ''; /** Access secret in S3-compatible external storage */ @Env('N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET') - readonly accessSecret: string = ''; + accessSecret: string = ''; } @Config class S3Config { /** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */ @Env('N8N_EXTERNAL_STORAGE_S3_HOST') - readonly host: string = ''; + host: string = ''; @Nested - readonly bucket: S3BucketConfig; + bucket: S3BucketConfig; @Nested - readonly credentials: S3CredentialsConfig; + credentials: S3CredentialsConfig; } @Config export class ExternalStorageConfig { @Nested - readonly s3: S3Config; + s3: S3Config; } diff --git a/packages/@n8n/config/src/configs/nodes.ts b/packages/@n8n/config/src/configs/nodes.config.ts similarity index 67% rename from packages/@n8n/config/src/configs/nodes.ts rename to packages/@n8n/config/src/configs/nodes.config.ts index f845607a8c..577c4055ab 100644 --- a/packages/@n8n/config/src/configs/nodes.ts +++ b/packages/@n8n/config/src/configs/nodes.config.ts @@ -25,22 +25,30 @@ class CommunityPackagesConfig { /** Whether to enable community packages */ @Env('N8N_COMMUNITY_PACKAGES_ENABLED') enabled: boolean = true; + + /** NPM registry URL to pull community packages from */ + @Env('N8N_COMMUNITY_PACKAGES_REGISTRY') + registry: string = 'https://registry.npmjs.org'; + + /** Whether to reinstall any missing community packages */ + @Env('N8N_REINSTALL_MISSING_PACKAGES') + reinstallMissing: boolean = false; } @Config export class NodesConfig { /** Node types to load. Includes all if unspecified. @example '["n8n-nodes-base.hackerNews"]' */ @Env('NODES_INCLUDE') - readonly include: JsonStringArray = []; + include: JsonStringArray = []; /** Node types not to load. Excludes none if unspecified. @example '["n8n-nodes-base.hackerNews"]' */ @Env('NODES_EXCLUDE') - readonly exclude: JsonStringArray = []; + exclude: JsonStringArray = []; /** Node type to use as error trigger */ @Env('NODES_ERROR_TRIGGER_TYPE') - readonly errorTriggerType: string = 'n8n-nodes-base.errorTrigger'; + errorTriggerType: string = 'n8n-nodes-base.errorTrigger'; @Nested - readonly communityPackages: CommunityPackagesConfig; + communityPackages: CommunityPackagesConfig; } diff --git a/packages/@n8n/config/src/configs/public-api.ts b/packages/@n8n/config/src/configs/public-api.config.ts similarity index 74% rename from packages/@n8n/config/src/configs/public-api.ts rename to packages/@n8n/config/src/configs/public-api.config.ts index 33e3bf3fc3..340c54fbd7 100644 --- a/packages/@n8n/config/src/configs/public-api.ts +++ b/packages/@n8n/config/src/configs/public-api.config.ts @@ -4,13 +4,13 @@ import { Config, Env } from '../decorators'; export class PublicApiConfig { /** Whether to disable the Public API */ @Env('N8N_PUBLIC_API_DISABLED') - readonly disabled: boolean = false; + disabled: boolean = false; /** Path segment for the Public API */ @Env('N8N_PUBLIC_API_ENDPOINT') - readonly path: string = 'api'; + path: string = 'api'; /** Whether to disable the Swagger UI for the Public API */ @Env('N8N_PUBLIC_API_SWAGGERUI_DISABLED') - readonly swaggerUiDisabled: boolean = false; + swaggerUiDisabled: boolean = false; } diff --git a/packages/@n8n/config/src/configs/scaling-mode.config.ts b/packages/@n8n/config/src/configs/scaling-mode.config.ts new file mode 100644 index 0000000000..750de77b07 --- /dev/null +++ b/packages/@n8n/config/src/configs/scaling-mode.config.ts @@ -0,0 +1,96 @@ +import { Config, Env, Nested } from '../decorators'; + +@Config +class HealthConfig { + /** Whether to enable the worker health check endpoint `/healthz`. */ + @Env('QUEUE_HEALTH_CHECK_ACTIVE') + active: boolean = false; + + /** Port for worker to respond to health checks requests on, if enabled. */ + @Env('QUEUE_HEALTH_CHECK_PORT') + port: number = 5678; +} + +@Config +class RedisConfig { + /** Redis database for Bull queue. */ + @Env('QUEUE_BULL_REDIS_DB') + db: number = 0; + + /** Redis host for Bull queue. */ + @Env('QUEUE_BULL_REDIS_HOST') + host: string = 'localhost'; + + /** Password to authenticate with Redis. */ + @Env('QUEUE_BULL_REDIS_PASSWORD') + password: string = ''; + + /** Port for Redis to listen on. */ + @Env('QUEUE_BULL_REDIS_PORT') + port: number = 6379; + + /** Max cumulative timeout (in milliseconds) of connection retries before process exit. */ + @Env('QUEUE_BULL_REDIS_TIMEOUT_THRESHOLD') + timeoutThreshold: number = 10_000; + + /** Redis username. Redis 6.0 or higher required. */ + @Env('QUEUE_BULL_REDIS_USERNAME') + username: string = ''; + + /** Redis cluster startup nodes, as comma-separated list of `{host}:{port}` pairs. @example 'redis-1:6379,redis-2:6379' */ + @Env('QUEUE_BULL_REDIS_CLUSTER_NODES') + clusterNodes: string = ''; + + /** Whether to enable TLS on Redis connections. */ + @Env('QUEUE_BULL_REDIS_TLS') + tls: boolean = false; +} + +@Config +class SettingsConfig { + /** How long (in milliseconds) is the lease period for a worker processing a job. */ + @Env('QUEUE_WORKER_LOCK_DURATION') + lockDuration: number = 30_000; + + /** How often (in milliseconds) a worker must renew the lease. */ + @Env('QUEUE_WORKER_LOCK_RENEW_TIME') + lockRenewTime: number = 15_000; + + /** How often (in milliseconds) Bull must check for stalled jobs. `0` to disable. */ + @Env('QUEUE_WORKER_STALLED_INTERVAL') + stalledInterval: number = 30_000; + + /** Max number of times a stalled job will be re-processed. See Bull's [documentation](https://docs.bullmq.io/guide/workers/stalled-jobs). */ + @Env('QUEUE_WORKER_MAX_STALLED_COUNT') + maxStalledCount: number = 1; +} + +@Config +class BullConfig { + /** Prefix for Bull keys on Redis. @example 'bull:jobs:23' */ + @Env('QUEUE_BULL_PREFIX') + prefix: string = 'bull'; + + @Nested + redis: RedisConfig; + + /** How often (in seconds) to poll the Bull queue to identify executions finished during a Redis crash. `0` to disable. May increase Redis traffic significantly. */ + @Env('QUEUE_RECOVERY_INTERVAL') + queueRecoveryInterval: number = 60; // watchdog interval + + /** @deprecated How long (in seconds) a worker must wait for active executions to finish before exiting. Use `N8N_GRACEFUL_SHUTDOWN_TIMEOUT` instead */ + @Env('QUEUE_WORKER_TIMEOUT') + gracefulShutdownTimeout: number = 30; + + @Nested + settings: SettingsConfig; +} + +@Config +export class ScalingModeConfig { + @Nested + health: HealthConfig; + + @Nested + bull: BullConfig; +} diff --git a/packages/@n8n/config/src/configs/templates.ts b/packages/@n8n/config/src/configs/templates.config.ts similarity index 74% rename from packages/@n8n/config/src/configs/templates.ts rename to packages/@n8n/config/src/configs/templates.config.ts index 3e10c892b3..0707330b32 100644 --- a/packages/@n8n/config/src/configs/templates.ts +++ b/packages/@n8n/config/src/configs/templates.config.ts @@ -4,9 +4,9 @@ import { Config, Env } from '../decorators'; export class TemplatesConfig { /** Whether to load workflow templates. */ @Env('N8N_TEMPLATES_ENABLED') - readonly enabled: boolean = true; + enabled: boolean = true; /** Host to retrieve workflow templates from endpoints. */ @Env('N8N_TEMPLATES_HOST') - readonly host: string = 'https://api.n8n.io/api/'; + host: string = 'https://api.n8n.io/api/'; } diff --git a/packages/@n8n/config/src/configs/email.ts b/packages/@n8n/config/src/configs/user-management.config.ts similarity index 66% rename from packages/@n8n/config/src/configs/email.ts rename to packages/@n8n/config/src/configs/user-management.config.ts index 318c352380..956bac2b75 100644 --- a/packages/@n8n/config/src/configs/email.ts +++ b/packages/@n8n/config/src/configs/user-management.config.ts @@ -1,78 +1,84 @@ import { Config, Env, Nested } from '../decorators'; @Config -export class SmtpAuth { +class SmtpAuth { /** SMTP login username */ @Env('N8N_SMTP_USER') - readonly user: string = ''; + user: string = ''; /** SMTP login password */ @Env('N8N_SMTP_PASS') - readonly pass: string = ''; + pass: string = ''; /** SMTP OAuth Service Client */ @Env('N8N_SMTP_OAUTH_SERVICE_CLIENT') - readonly serviceClient: string = ''; + serviceClient: string = ''; /** SMTP OAuth Private Key */ @Env('N8N_SMTP_OAUTH_PRIVATE_KEY') - readonly privateKey: string = ''; + privateKey: string = ''; } @Config -export class SmtpConfig { +class SmtpConfig { /** SMTP server host */ @Env('N8N_SMTP_HOST') - readonly host: string = ''; + host: string = ''; /** SMTP server port */ @Env('N8N_SMTP_PORT') - readonly port: number = 465; + port: number = 465; /** Whether to use SSL for SMTP */ @Env('N8N_SMTP_SSL') - readonly secure: boolean = true; + secure: boolean = true; /** Whether to use STARTTLS for SMTP when SSL is disabled */ @Env('N8N_SMTP_STARTTLS') - readonly startTLS: boolean = true; + startTLS: boolean = true; /** How to display sender name */ @Env('N8N_SMTP_SENDER') - readonly sender: string = ''; + sender: string = ''; @Nested - readonly auth: SmtpAuth; + auth: SmtpAuth; } @Config export class TemplateConfig { /** Overrides default HTML template for inviting new people (use full path) */ @Env('N8N_UM_EMAIL_TEMPLATES_INVITE') - readonly invite: string = ''; + invite: string = ''; /** Overrides default HTML template for resetting password (use full path) */ @Env('N8N_UM_EMAIL_TEMPLATES_PWRESET') - readonly passwordReset: string = ''; + passwordReset: string = ''; /** Overrides default HTML template for notifying that a workflow was shared (use full path) */ @Env('N8N_UM_EMAIL_TEMPLATES_WORKFLOW_SHARED') - readonly workflowShared: string = ''; + workflowShared: string = ''; /** Overrides default HTML template for notifying that credentials were shared (use full path) */ @Env('N8N_UM_EMAIL_TEMPLATES_CREDENTIALS_SHARED') - readonly credentialsShared: string = ''; + credentialsShared: string = ''; } @Config -export class EmailConfig { +class EmailConfig { /** How to send emails */ @Env('N8N_EMAIL_MODE') - readonly mode: '' | 'smtp' = 'smtp'; + mode: '' | 'smtp' = 'smtp'; @Nested - readonly smtp: SmtpConfig; + smtp: SmtpConfig; @Nested - readonly template: TemplateConfig; + template: TemplateConfig; +} + +@Config +export class UserManagementConfig { + @Nested + emails: EmailConfig; } diff --git a/packages/@n8n/config/src/configs/version-notifications.ts b/packages/@n8n/config/src/configs/version-notifications.config.ts similarity index 70% rename from packages/@n8n/config/src/configs/version-notifications.ts rename to packages/@n8n/config/src/configs/version-notifications.config.ts index 1aa693228d..313e264a2d 100644 --- a/packages/@n8n/config/src/configs/version-notifications.ts +++ b/packages/@n8n/config/src/configs/version-notifications.config.ts @@ -4,13 +4,13 @@ import { Config, Env } from '../decorators'; export class VersionNotificationsConfig { /** Whether to request notifications about new n8n versions */ @Env('N8N_VERSION_NOTIFICATIONS_ENABLED') - readonly enabled: boolean = true; + enabled: boolean = true; /** Endpoint to retrieve n8n version information from */ @Env('N8N_VERSION_NOTIFICATIONS_ENDPOINT') - readonly endpoint: string = 'https://api.n8n.io/api/versions/'; + endpoint: string = 'https://api.n8n.io/api/versions/'; /** URL for versions panel to page instructing user on how to update n8n instance */ @Env('N8N_VERSION_NOTIFICATIONS_INFO_URL') - readonly infoUrl: string = 'https://docs.n8n.io/hosting/installation/updating/'; + infoUrl: string = 'https://docs.n8n.io/hosting/installation/updating/'; } diff --git a/packages/@n8n/config/src/configs/workflows.ts b/packages/@n8n/config/src/configs/workflows.config.ts similarity index 60% rename from packages/@n8n/config/src/configs/workflows.ts rename to packages/@n8n/config/src/configs/workflows.config.ts index b19f4bc95d..3d6eaad12f 100644 --- a/packages/@n8n/config/src/configs/workflows.ts +++ b/packages/@n8n/config/src/configs/workflows.config.ts @@ -4,17 +4,14 @@ import { Config, Env } from '../decorators'; export class WorkflowsConfig { /** Default name for workflow */ @Env('WORKFLOWS_DEFAULT_NAME') - readonly defaultName: string = 'My workflow'; + defaultName: string = 'My workflow'; /** Show onboarding flow in new workflow */ @Env('N8N_ONBOARDING_FLOW_DISABLED') - readonly onboardingFlowDisabled: boolean = false; + onboardingFlowDisabled: boolean = false; /** Default option for which workflows may call the current workflow */ @Env('N8N_WORKFLOW_CALLER_POLICY_DEFAULT_OPTION') - readonly callerPolicyDefaultOption: - | 'any' - | 'none' - | 'workflowsFromAList' - | 'workflowsFromSameOwner' = 'workflowsFromSameOwner'; + callerPolicyDefaultOption: 'any' | 'none' | 'workflowsFromAList' | 'workflowsFromSameOwner' = + 'workflowsFromSameOwner'; } diff --git a/packages/@n8n/config/src/decorators.ts b/packages/@n8n/config/src/decorators.ts index fd68d44085..c5549a674f 100644 --- a/packages/@n8n/config/src/decorators.ts +++ b/packages/@n8n/config/src/decorators.ts @@ -47,6 +47,11 @@ export const Config: ClassDecorator = (ConfigClass: Class) => { } else { value = value === 'true'; } + } else if (type === Object) { + // eslint-disable-next-line n8n-local-rules/no-plain-errors + throw new Error( + `Invalid decorator metadata on key "${key as string}" on ${ConfigClass.name}\n Please use explicit typing on all config fields`, + ); } else if (type !== String && type !== Object) { value = new (type as Constructable)(value as string); } diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index b32c7c7dc3..fd1dc50c18 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -1,54 +1,80 @@ -import { Config, Nested } from './decorators'; -import { CredentialsConfig } from './configs/credentials'; -import { DatabaseConfig } from './configs/database'; -import { EmailConfig } from './configs/email'; -import { VersionNotificationsConfig } from './configs/version-notifications'; -import { PublicApiConfig } from './configs/public-api'; -import { ExternalSecretsConfig } from './configs/external-secrets'; -import { TemplatesConfig } from './configs/templates'; -import { EventBusConfig } from './configs/event-bus'; -import { NodesConfig } from './configs/nodes'; -import { ExternalStorageConfig } from './configs/external-storage'; -import { WorkflowsConfig } from './configs/workflows'; - -@Config -class UserManagementConfig { - @Nested - emails: EmailConfig; -} +import { Config, Env, Nested } from './decorators'; +import { CredentialsConfig } from './configs/credentials.config'; +import { DatabaseConfig } from './configs/database.config'; +import { VersionNotificationsConfig } from './configs/version-notifications.config'; +import { PublicApiConfig } from './configs/public-api.config'; +import { ExternalSecretsConfig } from './configs/external-secrets.config'; +import { TemplatesConfig } from './configs/templates.config'; +import { EventBusConfig } from './configs/event-bus.config'; +import { NodesConfig } from './configs/nodes.config'; +import { ExternalStorageConfig } from './configs/external-storage.config'; +import { WorkflowsConfig } from './configs/workflows.config'; +import { EndpointsConfig } from './configs/endpoints.config'; +import { CacheConfig } from './configs/cache.config'; +import { ScalingModeConfig } from './configs/scaling-mode.config'; +import { UserManagementConfig } from './configs/user-management.config'; @Config export class GlobalConfig { @Nested - readonly database: DatabaseConfig; + database: DatabaseConfig; @Nested - readonly credentials: CredentialsConfig; + credentials: CredentialsConfig; @Nested - readonly userManagement: UserManagementConfig; + userManagement: UserManagementConfig; @Nested - readonly versionNotifications: VersionNotificationsConfig; + versionNotifications: VersionNotificationsConfig; @Nested - readonly publicApi: PublicApiConfig; + publicApi: PublicApiConfig; @Nested - readonly externalSecrets: ExternalSecretsConfig; + externalSecrets: ExternalSecretsConfig; @Nested - readonly templates: TemplatesConfig; + templates: TemplatesConfig; @Nested - readonly eventBus: EventBusConfig; + eventBus: EventBusConfig; @Nested - readonly nodes: NodesConfig; + nodes: NodesConfig; @Nested - readonly externalStorage: ExternalStorageConfig; + externalStorage: ExternalStorageConfig; @Nested - readonly workflows: WorkflowsConfig; + workflows: WorkflowsConfig; + + /** Path n8n is deployed to */ + @Env('N8N_PATH') + path: string = '/'; + + /** Host name n8n can be reached */ + @Env('N8N_HOST') + host: string = 'localhost'; + + /** HTTP port n8n can be reached */ + @Env('N8N_PORT') + port: number = 5678; + + /** IP address n8n should listen on */ + @Env('N8N_LISTEN_ADDRESS') + listen_address: string = '0.0.0.0'; + + /** HTTP Protocol via which n8n can be reached */ + @Env('N8N_PROTOCOL') + protocol: 'http' | 'https' = 'http'; + + @Nested + endpoints: EndpointsConfig; + + @Nested + cache: CacheConfig; + + @Nested + queue: ScalingModeConfig; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 90c9ee3240..adecff7f9d 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -18,6 +18,11 @@ describe('GlobalConfig', () => { }); const defaultConfig: GlobalConfig = { + path: '/', + host: 'localhost', + port: 5678, + listen_address: '0.0.0.0', + protocol: 'http', database: { logging: { enabled: false, @@ -103,6 +108,8 @@ describe('GlobalConfig', () => { nodes: { communityPackages: { enabled: true, + registry: 'https://registry.npmjs.org', + reinstallMissing: false, }, errorTriggerType: 'n8n-nodes-base.errorTrigger', include: [], @@ -140,12 +147,82 @@ describe('GlobalConfig', () => { onboardingFlowDisabled: false, callerPolicyDefaultOption: 'workflowsFromSameOwner', }, + endpoints: { + metrics: { + enable: false, + prefix: 'n8n_', + includeWorkflowIdLabel: false, + includeDefaultMetrics: true, + includeMessageEventBusMetrics: false, + includeNodeTypeLabel: false, + includeCacheMetrics: false, + includeApiEndpoints: false, + includeApiPathLabel: false, + includeApiMethodLabel: false, + includeCredentialTypeLabel: false, + includeApiStatusCodeLabel: false, + }, + additionalNonUIRoutes: '', + disableProductionWebhooksOnMainProcess: false, + disableUi: false, + form: 'form', + formTest: 'form-test', + formWaiting: 'form-waiting', + payloadSizeMax: 16, + rest: 'rest', + webhook: 'webhook', + webhookTest: 'webhook-test', + webhookWaiting: 'webhook-waiting', + }, + cache: { + backend: 'auto', + memory: { + maxSize: 3145728, + ttl: 3600000, + }, + redis: { + prefix: 'redis', + ttl: 3600000, + }, + }, + queue: { + health: { + active: false, + port: 5678, + }, + bull: { + redis: { + db: 0, + host: 'localhost', + password: '', + port: 6379, + timeoutThreshold: 10_000, + username: '', + clusterNodes: '', + tls: false, + }, + queueRecoveryInterval: 60, + gracefulShutdownTimeout: 30, + prefix: 'bull', + settings: { + lockDuration: 30_000, + lockRenewTime: 15_000, + stalledInterval: 30_000, + maxStalledCount: 1, + }, + }, + }, }; it('should use all default values when no env variables are defined', () => { process.env = {}; const config = Container.get(GlobalConfig); - expect(config).toEqual(defaultConfig); + + // deepCopy for diff to show plain objects + // eslint-disable-next-line n8n-local-rules/no-json-parse-json-stringify + const deepCopy = (obj: T): T => JSON.parse(JSON.stringify(obj)); + + expect(deepCopy(config)).toEqual(defaultConfig); expect(mockFs.readFileSync).not.toHaveBeenCalled(); }); @@ -155,6 +232,7 @@ describe('GlobalConfig', () => { DB_POSTGRESDB_USER: 'n8n', DB_TABLE_PREFIX: 'test_', NODES_INCLUDE: '["n8n-nodes-base.hackerNews"]', + DB_LOGGING_MAX_EXECUTION_TIME: '0', }; const config = Container.get(GlobalConfig); expect(config).toEqual({ diff --git a/packages/@n8n/nodes-langchain/README.md b/packages/@n8n/nodes-langchain/README.md index e5761628cf..03a23d2186 100644 --- a/packages/@n8n/nodes-langchain/README.md +++ b/packages/@n8n/nodes-langchain/README.md @@ -8,6 +8,4 @@ These nodes are still in Beta state and are only compatible with the Docker imag ## License -n8n is [fair-code](https://faircode.io) distributed under the [**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md). - -Additional information about the license can be found in the [docs](https://docs.n8n.io/reference/license/). +You can find the license information [here](https://github.com/n8n-io/n8n/blob/master/README.md#license) diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts index 4a6d8a733c..62761742b0 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts @@ -9,7 +9,6 @@ import type { INodeTypeDescription, INodeProperties, } from 'n8n-workflow'; -import { getTemplateNoticeField } from '../../../utils/sharedFields'; import { promptTypeOptions, textInput } from '../../../utils/descriptions'; import { conversationalAgentProperties } from './agents/ConversationalAgent/description'; import { conversationalAgentExecute } from './agents/ConversationalAgent/execute'; @@ -83,6 +82,7 @@ function getInputs( filter: { nodes: [ '@n8n/n8n-nodes-langchain.lmChatAnthropic', + '@n8n/n8n-nodes-langchain.lmChatAwsBedrock', '@n8n/n8n-nodes-langchain.lmChatGroq', '@n8n/n8n-nodes-langchain.lmChatOllama', '@n8n/n8n-nodes-langchain.lmChatOpenAi', @@ -113,6 +113,7 @@ function getInputs( '@n8n/n8n-nodes-langchain.lmChatAnthropic', '@n8n/n8n-nodes-langchain.lmChatAzureOpenAi', '@n8n/n8n-nodes-langchain.lmChatMistralCloud', + '@n8n/n8n-nodes-langchain.lmChatOllama', '@n8n/n8n-nodes-langchain.lmChatOpenAi', '@n8n/n8n-nodes-langchain.lmChatGroq', '@n8n/n8n-nodes-langchain.lmChatGoogleVertex', @@ -206,35 +207,37 @@ const agentTypeProperty: INodeProperties = { name: 'Tools Agent', value: 'toolsAgent', description: - 'Utilized unified Tool calling interface to select the appropriate tools and argument for execution', + 'Utilizes structured tool schemas for precise and reliable tool selection and execution. Recommended for complex tasks requiring accurate and consistent tool usage, but only usable with models that support tool calling.', }, { name: 'Conversational Agent', value: 'conversationalAgent', description: - 'Selects tools to accomplish its task and uses memory to recall previous conversations', + 'Describes tools in the system prompt and parses JSON responses for tool calls. More flexible but potentially less reliable than the Tools Agent. Suitable for simpler interactions or with models not supporting structured schemas.', }, { name: 'OpenAI Functions Agent', value: 'openAiFunctionsAgent', description: - "Utilizes OpenAI's Function Calling feature to select the appropriate tool and arguments for execution", + "Leverages OpenAI's function calling capabilities to precisely select and execute tools. Excellent for tasks requiring structured outputs when working with OpenAI models.", }, { name: 'Plan and Execute Agent', value: 'planAndExecuteAgent', description: - 'Plan and execute agents accomplish an objective by first planning what to do, then executing the sub tasks', + 'Creates a high-level plan for complex tasks and then executes each step. Suitable for multi-stage problems or when a strategic approach is needed.', }, { name: 'ReAct Agent', value: 'reActAgent', - description: 'Strategically select tools to accomplish a given task', + description: + 'Combines reasoning and action in an iterative process. Effective for tasks that require careful analysis and step-by-step problem-solving.', }, { name: 'SQL Agent', value: 'sqlAgent', - description: 'Answers questions about data in an SQL database', + description: + 'Specializes in interacting with SQL databases. Ideal for data analysis tasks, generating queries, or extracting insights from structured data.', }, ], default: '', @@ -256,7 +259,7 @@ export class Agent implements INodeType { color: '#404040', }, codex: { - alias: ['LangChain'], + alias: ['LangChain', 'Chat', 'Conversational', 'Plan and Execute', 'ReAct', 'Tools'], categories: ['AI'], subcategories: { AI: ['Agents', 'Root Nodes'], @@ -302,10 +305,14 @@ export class Agent implements INodeType { ], properties: [ { - ...getTemplateNoticeField(1954), + displayName: + 'Tip: Get a feel for agents with our quick tutorial or see an example of how this node works', + name: 'notice_tip', + type: 'notice', + default: '', displayOptions: { show: { - agent: ['conversationalAgent'], + agent: ['conversationalAgent', 'toolsAgent'], }, }, }, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts index a027829e3a..3c4ff28f06 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts @@ -38,7 +38,7 @@ export async function openAiFunctionsAgentExecute( const memory = (await this.getInputConnectionData(NodeConnectionType.AiMemory, 0)) as | BaseChatMemory | undefined; - const tools = await getConnectedTools(this, nodeVersion >= 1.5); + const tools = await getConnectedTools(this, nodeVersion >= 1.5, false); const outputParsers = await getOptionalOutputParsers(this); const options = this.getNodeParameter('options', 0, {}) as { systemMessage?: string; diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts index 81ac6bad5d..e0da7f1e31 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts @@ -90,7 +90,7 @@ export async function toolsAgentExecute(this: IExecuteFunctions): Promise; + const tools = (await getConnectedTools(this, true, false)) as Array; const outputParser = (await getOptionalOutputParsers(this))?.[0]; let structuredOutputParserTool: DynamicStructuredTool | undefined; diff --git a/packages/@n8n/nodes-langchain/nodes/agents/OpenAiAssistant/OpenAiAssistant.node.ts b/packages/@n8n/nodes-langchain/nodes/agents/OpenAiAssistant/OpenAiAssistant.node.ts index f56ce7c5c4..8f05faccb0 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/OpenAiAssistant/OpenAiAssistant.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/OpenAiAssistant/OpenAiAssistant.node.ts @@ -313,7 +313,7 @@ export class OpenAiAssistant implements INodeType { async execute(this: IExecuteFunctions): Promise { const nodeVersion = this.getNode().typeVersion; - const tools = await getConnectedTools(this, nodeVersion > 1); + const tools = await getConnectedTools(this, nodeVersion > 1, false); const credentials = await this.getCredentials('openAiApi'); const items = this.getInputData(); diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts index 35b83dead8..003c8eb7dd 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts @@ -22,7 +22,7 @@ import { LLMChain } from 'langchain/chains'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { HumanMessage } from '@langchain/core/messages'; import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; -import { ChatOllama } from '@langchain/community/chat_models/ollama'; +import { ChatOllama } from '@langchain/ollama'; import { getTemplateNoticeField } from '../../../utils/sharedFields'; import { getOptionalOutputParsers, diff --git a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts new file mode 100644 index 0000000000..bdf5163d86 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts @@ -0,0 +1,308 @@ +import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; +import type { + INodeType, + INodeTypeDescription, + IExecuteFunctions, + INodeExecutionData, + INodePropertyOptions, +} from 'n8n-workflow'; +import type { JSONSchema7 } from 'json-schema'; +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts'; +import type { z } from 'zod'; +import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers'; +import { HumanMessage } from '@langchain/core/messages'; +import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; +import { + inputSchemaField, + jsonSchemaExampleField, + schemaTypeField, +} from '../../../utils/descriptions'; +import { getTracingConfig } from '../../../utils/tracing'; +import type { AttributeDefinition } from './types'; +import { makeZodSchemaFromAttributes } from './helpers'; + +const SYSTEM_PROMPT_TEMPLATE = `You are an expert extraction algorithm. +Only extract relevant information from the text. +If you do not know the value of an attribute asked to extract, you may omit the attribute's value.`; + +export class InformationExtractor implements INodeType { + description: INodeTypeDescription = { + displayName: 'Information Extractor', + name: 'informationExtractor', + icon: 'fa:project-diagram', + iconColor: 'black', + group: ['transform'], + version: 1, + description: 'Extract information from text in a structured format', + codex: { + alias: ['NER', 'parse', 'parsing', 'JSON', 'data extraction', 'structured'], + categories: ['AI'], + subcategories: { + AI: ['Chains', 'Root Nodes'], + }, + resources: { + primaryDocumentation: [ + { + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.information-extractor/', + }, + ], + }, + }, + defaults: { + name: 'Information Extractor', + }, + inputs: [ + { displayName: '', type: NodeConnectionType.Main }, + { + displayName: 'Model', + maxConnections: 1, + type: NodeConnectionType.AiLanguageModel, + required: true, + }, + ], + outputs: [NodeConnectionType.Main], + properties: [ + { + displayName: 'Text', + name: 'text', + type: 'string', + default: '', + description: 'The text to extract information from', + typeOptions: { + rows: 2, + }, + }, + { + ...schemaTypeField, + description: 'How to specify the schema for the desired output', + options: [ + { + name: 'From Attribute Descriptions', + value: 'fromAttributes', + description: + 'Extract specific attributes from the text based on types and descriptions', + } as INodePropertyOptions, + ...(schemaTypeField.options as INodePropertyOptions[]), + ], + default: 'fromAttributes', + }, + { + ...jsonSchemaExampleField, + default: `{ + "state": "California", + "cities": ["Los Angeles", "San Francisco", "San Diego"] +}`, + }, + { + ...inputSchemaField, + default: `{ + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "cities": { + "type": "array", + "items": { + "type": "string" + } + } + } +}`, + }, + { + displayName: + 'The schema has to be defined in the JSON Schema format. Look at this page for examples.', + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + schemaType: ['manual'], + }, + }, + }, + { + displayName: 'Attributes', + name: 'attributes', + placeholder: 'Add Attribute', + type: 'fixedCollection', + default: {}, + displayOptions: { + show: { + schemaType: ['fromAttributes'], + }, + }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + name: 'attributes', + displayName: 'Attribute List', + values: [ + { + displayName: 'Name', + name: 'name', + type: 'string', + default: '', + description: 'Attribute to extract', + placeholder: 'e.g. company_name', + required: true, + }, + { + displayName: 'Type', + name: 'type', + type: 'options', + description: 'Data type of the attribute', + required: true, + options: [ + { + name: 'Boolean', + value: 'boolean', + }, + { + name: 'Date', + value: 'date', + }, + { + name: 'Number', + value: 'number', + }, + { + name: 'String', + value: 'string', + }, + ], + default: 'string', + }, + { + displayName: 'Description', + name: 'description', + type: 'string', + default: '', + description: 'Describe your attribute', + placeholder: 'Add description for the attribute', + required: true, + }, + { + displayName: 'Required', + name: 'required', + type: 'boolean', + default: false, + description: 'Whether attribute is required', + required: true, + }, + ], + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + default: {}, + placeholder: 'Add Option', + options: [ + { + displayName: 'System Prompt Template', + name: 'systemPromptTemplate', + type: 'string', + default: SYSTEM_PROMPT_TEMPLATE, + description: 'String to use directly as the system prompt template', + typeOptions: { + rows: 6, + }, + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + + const llm = (await this.getInputConnectionData( + NodeConnectionType.AiLanguageModel, + 0, + )) as BaseLanguageModel; + + const schemaType = this.getNodeParameter('schemaType', 0, '') as + | 'fromAttributes' + | 'fromJson' + | 'manual'; + + let parser: OutputFixingParser; + + if (schemaType === 'fromAttributes') { + const attributes = this.getNodeParameter( + 'attributes.attributes', + 0, + [], + ) as AttributeDefinition[]; + + if (attributes.length === 0) { + throw new NodeOperationError(this.getNode(), 'At least one attribute must be specified'); + } + + parser = OutputFixingParser.fromLLM( + llm, + StructuredOutputParser.fromZodSchema(makeZodSchemaFromAttributes(attributes)), + ); + } else { + let jsonSchema: JSONSchema7; + + if (schemaType === 'fromJson') { + const jsonExample = this.getNodeParameter('jsonSchemaExample', 0, '') as string; + jsonSchema = generateSchema(jsonExample); + } else { + const inputSchema = this.getNodeParameter('inputSchema', 0, '') as string; + jsonSchema = jsonParse(inputSchema); + } + + const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); + const zodSchema = (await zodSchemaSandbox.runCode()) as z.ZodSchema; + + parser = OutputFixingParser.fromLLM(llm, StructuredOutputParser.fromZodSchema(zodSchema)); + } + + const resultData: INodeExecutionData[] = []; + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + const input = this.getNodeParameter('text', itemIndex) as string; + const inputPrompt = new HumanMessage(input); + + const options = this.getNodeParameter('options', itemIndex, {}) as { + systemPromptTemplate?: string; + }; + + const systemPromptTemplate = SystemMessagePromptTemplate.fromTemplate( + `${options.systemPromptTemplate ?? SYSTEM_PROMPT_TEMPLATE} +{format_instructions}`, + ); + + const messages = [ + await systemPromptTemplate.format({ + format_instructions: parser.getFormatInstructions(), + }), + inputPrompt, + ]; + const prompt = ChatPromptTemplate.fromMessages(messages); + const chain = prompt.pipe(llm).pipe(parser).withConfig(getTracingConfig(this)); + + try { + const output = await chain.invoke(messages); + resultData.push({ json: { output } }); + } catch (error) { + if (this.continueOnFail(error)) { + resultData.push({ json: { error: error.message }, pairedItem: { item: itemIndex } }); + continue; + } + + throw error; + } + } + + return [resultData]; + } +} diff --git a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/helpers.ts b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/helpers.ts new file mode 100644 index 0000000000..acde52765e --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/helpers.ts @@ -0,0 +1,33 @@ +import { z } from 'zod'; +import type { AttributeDefinition } from './types'; + +function makeAttributeSchema(attributeDefinition: AttributeDefinition, required: boolean = true) { + let schema: z.ZodTypeAny; + + if (attributeDefinition.type === 'string') { + schema = z.string(); + } else if (attributeDefinition.type === 'number') { + schema = z.number(); + } else if (attributeDefinition.type === 'boolean') { + schema = z.boolean(); + } else if (attributeDefinition.type === 'date') { + schema = z.string().date(); + } else { + schema = z.unknown(); + } + + if (!required) { + schema = schema.optional(); + } + + return schema.describe(attributeDefinition.description); +} + +export function makeZodSchemaFromAttributes(attributes: AttributeDefinition[]) { + const schemaEntries = attributes.map((attr) => [ + attr.name, + makeAttributeSchema(attr, attr.required), + ]); + + return z.object(Object.fromEntries(schemaEntries)); +} diff --git a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/test/InformationExtraction.node.test.ts b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/test/InformationExtraction.node.test.ts new file mode 100644 index 0000000000..b4e4672dac --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/test/InformationExtraction.node.test.ts @@ -0,0 +1,218 @@ +import type { IDataObject, IExecuteFunctions } from 'n8n-workflow/src'; +import get from 'lodash/get'; + +import { FakeLLM, FakeListChatModel } from '@langchain/core/utils/testing'; +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { InformationExtractor } from '../InformationExtractor.node'; +import { makeZodSchemaFromAttributes } from '../helpers'; +import type { AttributeDefinition } from '../types'; + +const mockPersonAttributes: AttributeDefinition[] = [ + { + name: 'name', + type: 'string', + description: 'The name of the person', + required: false, + }, + { + name: 'age', + type: 'number', + description: 'The age of the person', + required: false, + }, +]; + +const mockPersonAttributesRequired: AttributeDefinition[] = [ + { + name: 'name', + type: 'string', + description: 'The name of the person', + required: true, + }, + { + name: 'age', + type: 'number', + description: 'The age of the person', + required: true, + }, +]; + +function formatFakeLlmResponse(object: Record) { + return `\`\`\`json\n${JSON.stringify(object, null, 2)}\n\`\`\``; +} + +const createExecuteFunctionsMock = (parameters: IDataObject, fakeLlm: BaseLanguageModel) => { + const nodeParameters = parameters; + + return { + getNodeParameter(parameter: string) { + return get(nodeParameters, parameter); + }, + getNode() { + return {}; + }, + getInputConnectionData() { + return fakeLlm; + }, + getInputData() { + return [{ json: {} }]; + }, + getWorkflow() { + return { + name: 'Test Workflow', + }; + }, + getExecutionId() { + return 'test_execution_id'; + }, + continueOnFail() { + return false; + }, + } as unknown as IExecuteFunctions; +}; + +describe('InformationExtractor', () => { + describe('From Attribute Descriptions', () => { + it('should generate a schema from attribute descriptions with optional fields', async () => { + const schema = makeZodSchemaFromAttributes(mockPersonAttributes); + + expect(schema.parse({ name: 'John', age: 30 })).toEqual({ name: 'John', age: 30 }); + expect(schema.parse({ name: 'John' })).toEqual({ name: 'John' }); + expect(schema.parse({ age: 30 })).toEqual({ age: 30 }); + }); + + it('should make a request to LLM and return the extracted attributes', async () => { + const node = new InformationExtractor(); + + const response = await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributes, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeLLM({ response: formatFakeLlmResponse({ name: 'John', age: 30 }) }), + ), + ); + + expect(response).toEqual([[{ json: { output: { name: 'John', age: 30 } } }]]); + }); + + it('should not fail if LLM could not extract some attribute', async () => { + const node = new InformationExtractor(); + + const response = await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributes, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeLLM({ response: formatFakeLlmResponse({ name: 'John' }) }), + ), + ); + + expect(response).toEqual([[{ json: { output: { name: 'John' } } }]]); + }); + + it('should fail if LLM could not extract some required attribute', async () => { + const node = new InformationExtractor(); + + try { + await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributesRequired, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeLLM({ response: formatFakeLlmResponse({ name: 'John' }) }), + ), + ); + } catch (error) { + expect(error.message).toContain('Failed to parse'); + } + }); + + it('should fail if LLM extracted an attribute with the wrong type', async () => { + const node = new InformationExtractor(); + + try { + await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributes, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeLLM({ response: formatFakeLlmResponse({ name: 'John', age: '30' }) }), + ), + ); + } catch (error) { + expect(error.message).toContain('Failed to parse'); + } + }); + + it('retries if LLM fails to extract some required attribute', async () => { + const node = new InformationExtractor(); + + const response = await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributesRequired, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeListChatModel({ + responses: [ + formatFakeLlmResponse({ name: 'John' }), + formatFakeLlmResponse({ name: 'John', age: 30 }), + ], + }), + ), + ); + + expect(response).toEqual([[{ json: { output: { name: 'John', age: 30 } } }]]); + }); + + it('retries if LLM extracted an attribute with a wrong type', async () => { + const node = new InformationExtractor(); + + const response = await node.execute.call( + createExecuteFunctionsMock( + { + text: 'John is 30 years old', + attributes: { + attributes: mockPersonAttributesRequired, + }, + options: {}, + schemaType: 'fromAttributes', + }, + new FakeListChatModel({ + responses: [ + formatFakeLlmResponse({ name: 'John', age: '30' }), + formatFakeLlmResponse({ name: 'John', age: 30 }), + ], + }), + ), + ); + + expect(response).toEqual([[{ json: { output: { name: 'John', age: 30 } } }]]); + }); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/types.ts b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/types.ts new file mode 100644 index 0000000000..7e6f3a208d --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/types.ts @@ -0,0 +1,6 @@ +export interface AttributeDefinition { + name: string; + description: string; + type: 'string' | 'number' | 'boolean' | 'date'; + required: boolean; +} diff --git a/packages/@n8n/nodes-langchain/nodes/chains/SentimentAnalysis/SentimentAnalysis.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/SentimentAnalysis/SentimentAnalysis.node.ts new file mode 100644 index 0000000000..6f627edcfd --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/SentimentAnalysis/SentimentAnalysis.node.ts @@ -0,0 +1,257 @@ +import type { + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeParameters, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; + +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { HumanMessage } from '@langchain/core/messages'; +import { SystemMessagePromptTemplate, ChatPromptTemplate } from '@langchain/core/prompts'; +import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers'; +import { z } from 'zod'; +import { getTracingConfig } from '../../../utils/tracing'; + +const DEFAULT_SYSTEM_PROMPT_TEMPLATE = + 'You are highly intelligent and accurate sentiment analyzer. Analyze the sentiment of the provided text. Categorize it into one of the following: {categories}. Use the provided formatting instructions. Only output the JSON.'; + +const DEFAULT_CATEGORIES = 'Positive, Neutral, Negative'; +const configuredOutputs = (parameters: INodeParameters, defaultCategories: string) => { + const options = (parameters?.options ?? {}) as IDataObject; + const categories = (options?.categories as string) ?? defaultCategories; + const categoriesArray = categories.split(',').map((cat) => cat.trim()); + + const ret = categoriesArray.map((cat) => ({ type: NodeConnectionType.Main, displayName: cat })); + return ret; +}; + +export class SentimentAnalysis implements INodeType { + description: INodeTypeDescription = { + displayName: 'Sentiment Analysis', + name: 'sentimentAnalysis', + icon: 'fa:balance-scale-left', + iconColor: 'black', + group: ['transform'], + version: 1, + description: 'Analyze the sentiment of your text', + codex: { + categories: ['AI'], + subcategories: { + AI: ['Chains', 'Root Nodes'], + }, + resources: { + primaryDocumentation: [ + { + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.sentimentanalysis/', + }, + ], + }, + }, + defaults: { + name: 'Sentiment Analysis', + }, + inputs: [ + { displayName: '', type: NodeConnectionType.Main }, + { + displayName: 'Model', + maxConnections: 1, + type: NodeConnectionType.AiLanguageModel, + required: true, + }, + ], + outputs: `={{(${configuredOutputs})($parameter, "${DEFAULT_CATEGORIES}")}}`, + properties: [ + { + displayName: 'Text to Analyze', + name: 'inputText', + type: 'string', + required: true, + default: '', + description: 'Use an expression to reference data in previous nodes or enter static text', + typeOptions: { + rows: 2, + }, + }, + { + displayName: + 'Sentiment scores are LLM-generated estimates, not statistically rigorous measurements. They may be inconsistent across runs and should be used as rough indicators only.', + name: 'detailedResultsNotice', + type: 'notice', + default: '', + displayOptions: { + show: { + '/options.includeDetailedResults': [true], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + default: {}, + placeholder: 'Add Option', + options: [ + { + displayName: 'Sentiment Categories', + name: 'categories', + type: 'string', + default: DEFAULT_CATEGORIES, + description: 'A comma-separated list of categories to analyze', + noDataExpression: true, + typeOptions: { + rows: 2, + }, + }, + { + displayName: 'System Prompt Template', + name: 'systemPromptTemplate', + type: 'string', + default: DEFAULT_SYSTEM_PROMPT_TEMPLATE, + description: 'String to use directly as the system prompt template', + typeOptions: { + rows: 6, + }, + }, + { + displayName: 'Include Detailed Results', + name: 'includeDetailedResults', + type: 'boolean', + default: false, + description: + 'Whether to include sentiment strength and confidence scores in the output', + }, + { + displayName: 'Enable Auto-Fixing', + name: 'enableAutoFixing', + type: 'boolean', + default: true, + description: 'Whether to enable auto-fixing for the output parser', + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + + const llm = (await this.getInputConnectionData( + NodeConnectionType.AiLanguageModel, + 0, + )) as BaseLanguageModel; + + const returnData: INodeExecutionData[][] = []; + + for (let i = 0; i < items.length; i++) { + try { + const sentimentCategories = this.getNodeParameter( + 'options.categories', + i, + DEFAULT_CATEGORIES, + ) as string; + + const categories = sentimentCategories + .split(',') + .map((cat) => cat.trim()) + .filter(Boolean); + + if (categories.length === 0) { + throw new NodeOperationError(this.getNode(), 'No sentiment categories provided', { + itemIndex: i, + }); + } + + // Initialize returnData with empty arrays for each category + if (returnData.length === 0) { + returnData.push(...Array.from({ length: categories.length }, () => [])); + } + + const options = this.getNodeParameter('options', i, {}) as { + systemPromptTemplate?: string; + includeDetailedResults?: boolean; + enableAutoFixing?: boolean; + }; + + const schema = z.object({ + sentiment: z.enum(categories as [string, ...string[]]), + strength: z + .number() + .min(0) + .max(1) + .describe('Strength score for sentiment in relation to the category'), + confidence: z.number().min(0).max(1), + }); + + const structuredParser = StructuredOutputParser.fromZodSchema(schema); + + const parser = options.enableAutoFixing + ? OutputFixingParser.fromLLM(llm, structuredParser) + : structuredParser; + + const systemPromptTemplate = SystemMessagePromptTemplate.fromTemplate( + `${options.systemPromptTemplate ?? DEFAULT_SYSTEM_PROMPT_TEMPLATE} + {format_instructions}`, + ); + + const input = this.getNodeParameter('inputText', i) as string; + const inputPrompt = new HumanMessage(input); + const messages = [ + await systemPromptTemplate.format({ + categories: sentimentCategories, + format_instructions: parser.getFormatInstructions(), + }), + inputPrompt, + ]; + + const prompt = ChatPromptTemplate.fromMessages(messages); + const chain = prompt.pipe(llm).pipe(parser).withConfig(getTracingConfig(this)); + + try { + const output = await chain.invoke(messages); + const sentimentIndex = categories.findIndex( + (s) => s.toLowerCase() === output.sentiment.toLowerCase(), + ); + + if (sentimentIndex !== -1) { + const resultItem = { ...items[i] }; + const sentimentAnalysis: IDataObject = { + category: output.sentiment, + }; + if (options.includeDetailedResults) { + sentimentAnalysis.strength = output.strength; + sentimentAnalysis.confidence = output.confidence; + } + resultItem.json = { + ...resultItem.json, + sentimentAnalysis, + }; + returnData[sentimentIndex].push(resultItem); + } + } catch (error) { + throw new NodeOperationError( + this.getNode(), + 'Error during parsing of LLM output, please check your LLM model and configuration', + { + itemIndex: i, + }, + ); + } + } catch (error) { + if (this.continueOnFail(error)) { + const executionErrorData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ error: error.message }), + { itemData: { item: i } }, + ); + returnData[0].push(...executionErrorData); + continue; + } + throw error; + } + } + return returnData; + } +} diff --git a/packages/@n8n/nodes-langchain/nodes/chains/TextClassifier/TextClassifier.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/TextClassifier/TextClassifier.node.ts index 6a433eaac3..0033361a07 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/TextClassifier/TextClassifier.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/TextClassifier/TextClassifier.node.ts @@ -46,7 +46,7 @@ export class TextClassifier implements INodeType { resources: { primaryDocumentation: [ { - url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.chainllm/', + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.text-classifier/', }, ], }, @@ -203,20 +203,27 @@ export class TextClassifier implements INodeType { discard: 'If there is not a very fitting category, select none of the categories.', }[fallback]; - const systemPromptTemplate = SystemMessagePromptTemplate.fromTemplate( - `${options.systemPromptTemplate ?? SYSTEM_PROMPT_TEMPLATE} -{format_instructions} -${multiClassPrompt} -${fallbackPrompt}`, - ); - const returnData: INodeExecutionData[][] = Array.from( { length: categories.length + (fallback === 'other' ? 1 : 0) }, (_) => [], ); for (let itemIdx = 0; itemIdx < items.length; itemIdx++) { + const item = items[itemIdx]; + item.pairedItem = { item: itemIdx }; const input = this.getNodeParameter('inputText', itemIdx) as string; const inputPrompt = new HumanMessage(input); + + const systemPromptTemplateOpt = this.getNodeParameter( + 'options.systemPromptTemplate', + itemIdx, + ) as string; + const systemPromptTemplate = SystemMessagePromptTemplate.fromTemplate( + `${systemPromptTemplateOpt ?? SYSTEM_PROMPT_TEMPLATE} +{format_instructions} +${multiClassPrompt} +${fallbackPrompt}`, + ); + const messages = [ await systemPromptTemplate.format({ categories: categories.map((cat) => cat.category).join(', '), @@ -227,13 +234,27 @@ ${fallbackPrompt}`, const prompt = ChatPromptTemplate.fromMessages(messages); const chain = prompt.pipe(llm).pipe(parser).withConfig(getTracingConfig(this)); - const output = await chain.invoke(messages); - categories.forEach((cat, idx) => { - if (output[cat.category]) returnData[idx].push(items[itemIdx]); - }); - if (fallback === 'other' && output.fallback) - returnData[returnData.length - 1].push(items[itemIdx]); + try { + const output = await chain.invoke(messages); + + categories.forEach((cat, idx) => { + if (output[cat.category]) returnData[idx].push(item); + }); + if (fallback === 'other' && output.fallback) returnData[returnData.length - 1].push(item); + } catch (error) { + if (this.continueOnFail(error)) { + returnData[0].push({ + json: { error: error.message }, + pairedItem: { item: itemIdx }, + }); + + continue; + } + + throw error; + } } + return returnData; } } diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts index 241efadb6d..ecc14e1344 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.ts @@ -74,7 +74,7 @@ export class LmChatAnthropic implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts index de6a53bfc4..b4fc474dd2 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOllama/LmChatOllama.node.ts @@ -7,8 +7,8 @@ import { type SupplyData, } from 'n8n-workflow'; -import type { ChatOllamaInput } from '@langchain/community/chat_models/ollama'; -import { ChatOllama } from '@langchain/community/chat_models/ollama'; +import type { ChatOllamaInput } from '@langchain/ollama'; +import { ChatOllama } from '@langchain/ollama'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; import { ollamaModel, ollamaOptions, ollamaDescription } from '../LMOllama/description'; import { N8nLlmTracing } from '../N8nLlmTracing'; @@ -28,7 +28,7 @@ export class LmChatOllama implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts index 264c594d1b..1f39c08229 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts @@ -26,7 +26,7 @@ export class LmChatOpenAi implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts index ee5163a78f..191209bb33 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMCohere/LmCohere.node.ts @@ -26,7 +26,7 @@ export class LmCohere implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Text Completion Models'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts index 95024ad4b2..5492a51a97 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOllama/LmOllama.node.ts @@ -27,7 +27,7 @@ export class LmOllama implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Text Completion Models'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts index 2f0e3480d8..a46ad429a2 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenAi/LmOpenAi.node.ts @@ -38,7 +38,7 @@ export class LmOpenAi implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Text Completion Models'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts index 9a30ef74d7..7b2c821f9c 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMOpenHuggingFaceInference/LmOpenHuggingFaceInference.node.ts @@ -26,7 +26,7 @@ export class LmOpenHuggingFaceInference implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Text Completion Models'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts index d313ba53b5..4e1c27bfde 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts @@ -29,7 +29,7 @@ export class LmChatAwsBedrock implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts index 2e05b4770d..5770387158 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.ts @@ -27,7 +27,7 @@ export class LmChatAzureOpenAi implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts index d3d3d1ea2c..ce08a650f2 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts @@ -27,7 +27,7 @@ export class LmChatGoogleGemini implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGooglePalm/LmChatGooglePalm.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGooglePalm/LmChatGooglePalm.node.ts index a32a5e959c..6195d4d987 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGooglePalm/LmChatGooglePalm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGooglePalm/LmChatGooglePalm.node.ts @@ -25,7 +25,7 @@ export class LmChatGooglePalm implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts index 1e3837818f..044428c01a 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts @@ -32,7 +32,7 @@ export class LmChatGoogleVertex implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts index 3354ac030f..d0a28715e1 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGroq/LmChatGroq.node.ts @@ -26,7 +26,7 @@ export class LmChatGroq implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts index 32364545d2..129beeadfe 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.ts @@ -27,7 +27,7 @@ export class LmChatMistralCloud implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Chat Models (Recommended)'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmGooglePalm/LmGooglePalm.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmGooglePalm/LmGooglePalm.node.ts index d79f74be02..e4681803fe 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmGooglePalm/LmGooglePalm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmGooglePalm/LmGooglePalm.node.ts @@ -25,7 +25,7 @@ export class LmGooglePalm implements INodeType { codex: { categories: ['AI'], subcategories: { - AI: ['Language Models'], + AI: ['Language Models', 'Root Nodes'], 'Language Models': ['Text Completion Models'], }, resources: { diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts index 2b7e205de6..b8eea7a5e2 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryBufferWindow/MemoryBufferWindow.node.ts @@ -10,7 +10,7 @@ import type { BufferWindowMemoryInput } from 'langchain/memory'; import { BufferWindowMemory } from 'langchain/memory'; import { logWrapper } from '../../../utils/logWrapper'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { sessionIdOption, sessionKeyProperty } from '../descriptions'; +import { sessionIdOption, sessionKeyProperty, contextWindowLengthProperty } from '../descriptions'; import { getSessionId } from '../../../utils/helpers'; class MemoryChatBufferSingleton { @@ -130,13 +130,7 @@ export class MemoryBufferWindow implements INodeType { }, }, sessionKeyProperty, - { - displayName: 'Context Window Length', - name: 'contextWindowLength', - type: 'number', - default: 5, - description: 'The number of previous messages to consider for context', - }, + contextWindowLengthProperty, ], }; diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts index ea3ed3c33e..b1a9cd7aea 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts @@ -1,7 +1,7 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import type { IExecuteFunctions, INodeType, INodeTypeDescription, SupplyData } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; -import { BufferMemory } from 'langchain/memory'; +import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; import { PostgresChatMessageHistory } from '@langchain/community/stores/message/postgres'; import type pg from 'pg'; import { configurePostgres } from 'n8n-nodes-base/dist/nodes/Postgres/v2/transport'; @@ -9,7 +9,7 @@ import type { PostgresNodeCredentials } from 'n8n-nodes-base/dist/nodes/Postgres import { postgresConnectionTest } from 'n8n-nodes-base/dist/nodes/Postgres/v2/methods/credentialTest'; import { logWrapper } from '../../../utils/logWrapper'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { sessionIdOption, sessionKeyProperty } from '../descriptions'; +import { sessionIdOption, sessionKeyProperty, contextWindowLengthProperty } from '../descriptions'; import { getSessionId } from '../../../utils/helpers'; export class MemoryPostgresChat implements INodeType { @@ -18,7 +18,7 @@ export class MemoryPostgresChat implements INodeType { name: 'memoryPostgresChat', icon: 'file:postgres.svg', group: ['transform'], - version: [1], + version: [1, 1.1], description: 'Stores the chat history in Postgres table.', defaults: { name: 'Postgres Chat Memory', @@ -60,6 +60,10 @@ export class MemoryPostgresChat implements INodeType { description: 'The table name to store the chat history in. If table does not exist, it will be created.', }, + { + ...contextWindowLengthProperty, + displayOptions: { hide: { '@version': [{ _cnd: { lt: 1.1 } }] } }, + }, ], }; @@ -83,12 +87,19 @@ export class MemoryPostgresChat implements INodeType { tableName, }); - const memory = new BufferMemory({ + const memClass = this.getNode().typeVersion < 1.1 ? BufferMemory : BufferWindowMemory; + const kOptions = + this.getNode().typeVersion < 1.1 + ? {} + : { k: this.getNodeParameter('contextWindowLength', itemIndex) }; + + const memory = new memClass({ memoryKey: 'chat_history', chatHistory: pgChatHistory, returnMessages: true, inputKey: 'input', outputKey: 'output', + ...kOptions, }); async function closeFunction() { diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts index d139bd31e3..da57ede1d2 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryRedisChat/MemoryRedisChat.node.ts @@ -7,14 +7,14 @@ import { type SupplyData, NodeConnectionType, } from 'n8n-workflow'; -import { BufferMemory } from 'langchain/memory'; +import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; import type { RedisChatMessageHistoryInput } from '@langchain/redis'; import { RedisChatMessageHistory } from '@langchain/redis'; import type { RedisClientOptions } from 'redis'; import { createClient } from 'redis'; import { logWrapper } from '../../../utils/logWrapper'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { sessionIdOption, sessionKeyProperty } from '../descriptions'; +import { sessionIdOption, sessionKeyProperty, contextWindowLengthProperty } from '../descriptions'; import { getSessionId } from '../../../utils/helpers'; export class MemoryRedisChat implements INodeType { @@ -23,7 +23,7 @@ export class MemoryRedisChat implements INodeType { name: 'memoryRedisChat', icon: 'file:redis.svg', group: ['transform'], - version: [1, 1.1, 1.2], + version: [1, 1.1, 1.2, 1.3], description: 'Stores the chat history in Redis.', defaults: { name: 'Redis Chat Memory', @@ -95,6 +95,10 @@ export class MemoryRedisChat implements INodeType { description: 'For how long the session should be stored in seconds. If set to 0 it will not expire.', }, + { + ...contextWindowLengthProperty, + displayOptions: { hide: { '@version': [{ _cnd: { lt: 1.3 } }] } }, + }, ], }; @@ -143,12 +147,19 @@ export class MemoryRedisChat implements INodeType { } const redisChatHistory = new RedisChatMessageHistory(redisChatConfig); - const memory = new BufferMemory({ + const memClass = this.getNode().typeVersion < 1.3 ? BufferMemory : BufferWindowMemory; + const kOptions = + this.getNode().typeVersion < 1.3 + ? {} + : { k: this.getNodeParameter('contextWindowLength', itemIndex) }; + + const memory = new memClass({ memoryKey: 'chat_history', chatHistory: redisChatHistory, returnMessages: true, inputKey: 'input', outputKey: 'output', + ...kOptions, }); async function closeFunction() { diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts index e5c9dc4c35..f0177d9e75 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryXata/MemoryXata.node.ts @@ -2,11 +2,11 @@ import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import type { IExecuteFunctions, INodeType, INodeTypeDescription, SupplyData } from 'n8n-workflow'; import { XataChatMessageHistory } from '@langchain/community/stores/message/xata'; -import { BufferMemory } from 'langchain/memory'; +import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; import { BaseClient } from '@xata.io/client'; import { logWrapper } from '../../../utils/logWrapper'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; -import { sessionIdOption, sessionKeyProperty } from '../descriptions'; +import { sessionIdOption, sessionKeyProperty, contextWindowLengthProperty } from '../descriptions'; import { getSessionId } from '../../../utils/helpers'; export class MemoryXata implements INodeType { @@ -15,7 +15,7 @@ export class MemoryXata implements INodeType { name: 'memoryXata', icon: 'file:xata.svg', group: ['transform'], - version: [1, 1.1, 1.2], + version: [1, 1.1, 1.2, 1.3], description: 'Use Xata Memory', defaults: { name: 'Xata', @@ -81,6 +81,10 @@ export class MemoryXata implements INodeType { }, }, sessionKeyProperty, + { + ...contextWindowLengthProperty, + displayOptions: { hide: { '@version': [{ _cnd: { lt: 1.3 } }] } }, + }, ], }; @@ -120,12 +124,19 @@ export class MemoryXata implements INodeType { apiKey: credentials.apiKey as string, }); - const memory = new BufferMemory({ + const memClass = this.getNode().typeVersion < 1.3 ? BufferMemory : BufferWindowMemory; + const kOptions = + this.getNode().typeVersion < 1.3 + ? {} + : { k: this.getNodeParameter('contextWindowLength', itemIndex) }; + + const memory = new memClass({ chatHistory, memoryKey: 'chat_history', returnMessages: true, inputKey: 'input', outputKey: 'output', + ...kOptions, }); return { diff --git a/packages/@n8n/nodes-langchain/nodes/memory/descriptions.ts b/packages/@n8n/nodes-langchain/nodes/memory/descriptions.ts index 5f722c4647..354d134fb7 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/descriptions.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/descriptions.ts @@ -33,3 +33,11 @@ export const sessionKeyProperty: INodeProperties = { }, }, }; + +export const contextWindowLengthProperty: INodeProperties = { + displayName: 'Context Window Length', + name: 'contextWindowLength', + type: 'number', + default: 5, + hint: 'How many past interactions the model receives as context', +}; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts index 492284b190..6ae5a88075 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts @@ -6,6 +6,7 @@ import type { SupplyData, ExecutionError, } from 'n8n-workflow'; + import { NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox'; @@ -208,7 +209,7 @@ export class ToolCode implements INodeType { try { response = await runFunction(query); } catch (error: unknown) { - executionError = error as ExecutionError; + executionError = new NodeOperationError(this.getNode(), error as ExecutionError); response = `There was an error: "${executionError.message}"`; } @@ -229,6 +230,7 @@ export class ToolCode implements INodeType { } else { void this.addOutputData(NodeConnectionType.AiTool, index, [[{ json: { response } }]]); } + return response; }, }), diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts index 1d391f313e..421e85e1b5 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts @@ -12,6 +12,7 @@ import { NodeConnectionType, NodeOperationError, tryToParseAlphanumericString } import { DynamicTool } from '@langchain/core/tools'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; +import { N8nTool } from '../../../utils/N8nTool'; import { configureHttpRequestFunction, configureResponseOptimizer, @@ -19,6 +20,7 @@ import { prepareToolDescription, configureToolFunction, updateParametersAndOptions, + makeToolInputSchema, } from './utils'; import { @@ -38,7 +40,7 @@ export class ToolHttpRequest implements INodeType { name: 'toolHttpRequest', icon: { light: 'file:httprequest.svg', dark: 'file:httprequest.dark.svg' }, group: ['output'], - version: 1, + version: [1, 1.1], description: 'Makes an HTTP request and returns the response data', subtitle: '={{ $parameter.toolDescription }}', defaults: { @@ -394,9 +396,24 @@ export class ToolHttpRequest implements INodeType { optimizeResponse, ); - const description = prepareToolDescription(toolDescription, toolParameters); + let tool: DynamicTool | N8nTool; - const tool = new DynamicTool({ name, description, func }); + // If the node version is 1.1 or higher, we use the N8nTool wrapper: + // it allows to use tool as a DynamicStructuredTool and have a fallback to DynamicTool + if (this.getNode().typeVersion >= 1.1) { + const schema = makeToolInputSchema(toolParameters); + + tool = new N8nTool(this, { + name, + description: toolDescription, + func, + schema, + }); + } else { + // Keep the old behavior for nodes with version 1.0 + const description = prepareToolDescription(toolDescription, toolParameters); + tool = new DynamicTool({ name, description, func }); + } return { response: tool, diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts index 28015b588a..96ae0d8492 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts @@ -27,6 +27,8 @@ import type { SendIn, ToolParameter, } from './interfaces'; +import type { DynamicZodObject } from '../../../types/zod.types'; +import { z } from 'zod'; const genericCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: number) => { const genericType = ctx.getNodeParameter('genericAuthType', itemIndex) as string; @@ -566,7 +568,7 @@ export const configureToolFunction = ( httpRequest: (options: IHttpRequestOptions) => Promise, optimizeResponse: (response: string) => string, ) => { - return async (query: string): Promise => { + return async (query: string | IDataObject): Promise => { const { index } = ctx.addInputData(NodeConnectionType.AiTool, [[{ json: { query } }]]); let response: string = ''; @@ -581,18 +583,22 @@ export const configureToolFunction = ( if (query) { let dataFromModel; - try { - dataFromModel = jsonParse(query); - } catch (error) { - if (toolParameters.length === 1) { - dataFromModel = { [toolParameters[0].name]: query }; - } else { - throw new NodeOperationError( - ctx.getNode(), - `Input is not a valid JSON: ${error.message}`, - { itemIndex }, - ); + if (typeof query === 'string') { + try { + dataFromModel = jsonParse(query); + } catch (error) { + if (toolParameters.length === 1) { + dataFromModel = { [toolParameters[0].name]: query }; + } else { + throw new NodeOperationError( + ctx.getNode(), + `Input is not a valid JSON: ${error.message}`, + { itemIndex }, + ); + } } + } else { + dataFromModel = query; } for (const parameter of toolParameters) { @@ -727,6 +733,8 @@ export const configureToolFunction = ( } } } catch (error) { + console.error(error); + const errorMessage = 'Input provided by model is not valid'; if (error instanceof NodeOperationError) { @@ -765,3 +773,38 @@ export const configureToolFunction = ( return response; }; }; + +function makeParameterZodSchema(parameter: ToolParameter) { + let schema: z.ZodTypeAny; + + if (parameter.type === 'string') { + schema = z.string(); + } else if (parameter.type === 'number') { + schema = z.number(); + } else if (parameter.type === 'boolean') { + schema = z.boolean(); + } else if (parameter.type === 'json') { + schema = z.record(z.any()); + } else { + schema = z.string(); + } + + if (!parameter.required) { + schema = schema.optional(); + } + + if (parameter.description) { + schema = schema.describe(parameter.description); + } + + return schema; +} + +export function makeToolInputSchema(parameters: ToolParameter[]): DynamicZodObject { + const schemaEntries = parameters.map((parameter) => [ + parameter.name, + makeParameterZodSchema(parameter), + ]); + + return z.object(Object.fromEntries(schemaEntries)); +} diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts index 54b8318f5b..5ed96cbd60 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts @@ -493,7 +493,7 @@ export class ToolWorkflow implements INodeType { if (useSchema) { try { // We initialize these even though one of them will always be empty - // it makes it easer to navigate the ternary operator + // it makes it easier to navigate the ternary operator const jsonExample = this.getNodeParameter('jsonSchemaExample', itemIndex, '') as string; const inputSchema = this.getNodeParameter('inputSchema', itemIndex, '') as string; diff --git a/packages/@n8n/nodes-langchain/nodes/trigger/ManualChatTrigger/ManualChatTrigger.node.ts b/packages/@n8n/nodes-langchain/nodes/trigger/ManualChatTrigger/ManualChatTrigger.node.ts index 272213d539..816b56b59a 100644 --- a/packages/@n8n/nodes-langchain/nodes/trigger/ManualChatTrigger/ManualChatTrigger.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/trigger/ManualChatTrigger/ManualChatTrigger.node.ts @@ -50,7 +50,9 @@ export class ManualChatTrigger implements INodeType { name: 'openChat', type: 'button', typeOptions: { - action: 'openChat', + buttonConfig: { + action: 'openChat', + }, }, default: '', }, diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/assistant/message.operation.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/assistant/message.operation.ts index da2770d05f..3ec2e46eea 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/assistant/message.operation.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/assistant/message.operation.ts @@ -4,7 +4,12 @@ import { OpenAIAssistantRunnable } from 'langchain/experimental/openai_assistant import type { OpenAIToolType } from 'langchain/dist/experimental/openai_assistant/schema'; import { OpenAI as OpenAIClient } from 'openai'; -import { NodeConnectionType, NodeOperationError, updateDisplayOptions } from 'n8n-workflow'; +import { + ApplicationError, + NodeConnectionType, + NodeOperationError, + updateDisplayOptions, +} from 'n8n-workflow'; import type { IDataObject, IExecuteFunctions, @@ -163,7 +168,7 @@ export async function execute(this: IExecuteFunctions, i: number): Promise 1); + const tools = await getConnectedTools(this, nodeVersion > 1, false); let assistantTools; if (tools.length) { @@ -228,25 +233,36 @@ export async function execute(this: IExecuteFunctions, i: number): Promise= 1.3 && + (assistantTools ?? [])?.length + ) { + await client.beta.assistants.update(assistantId, { + tools: assistantTools, + }); + } + filteredResponse = omit(response, ['signal', 'timeout']) as IDataObject; + } catch (error) { + if (!(error instanceof ApplicationError)) { + throw new NodeOperationError(this.getNode(), error.message, { itemIndex: i }); } } - if ( - options.preserveOriginalTools !== false && - nodeVersion >= 1.3 && - (assistantTools ?? [])?.length - ) { - await client.beta.assistants.update(assistantId, { - tools: assistantTools, - }); - } - const filteredResponse = omit(response, ['signal', 'timeout']); return [{ json: filteredResponse, pairedItem: { item: i } }]; } diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/text/message.operation.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/text/message.operation.ts index 4cf72e9f5f..d37be5a065 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/text/message.operation.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/actions/text/message.operation.ts @@ -219,7 +219,7 @@ export async function execute(this: IExecuteFunctions, i: number): Promise 1; - externalTools = await getConnectedTools(this, enforceUniqueNames); + externalTools = await getConnectedTools(this, enforceUniqueNames, false); } if (externalTools.length) { diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index 8513d6ea62..28303136c2 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/n8n-nodes-langchain", - "version": "1.52.0", + "version": "1.56.0", "description": "", "main": "index.js", "scripts": { @@ -45,6 +45,8 @@ "dist/nodes/chains/ChainSummarization/ChainSummarization.node.js", "dist/nodes/chains/ChainLLM/ChainLlm.node.js", "dist/nodes/chains/ChainRetrievalQA/ChainRetrievalQa.node.js", + "dist/nodes/chains/SentimentAnalysis/SentimentAnalysis.node.js", + "dist/nodes/chains/InformationExtractor/InformationExtractor.node.js", "dist/nodes/chains/TextClassifier/TextClassifier.node.js", "dist/nodes/code/Code.node.js", "dist/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.js", @@ -119,7 +121,7 @@ ] }, "devDependencies": { - "@types/basic-auth": "^1.1.3", + "@types/basic-auth": "catalog:", "@types/cheerio": "^0.22.15", "@types/html-to-text": "^9.0.1", "@types/json-schema": "^7.0.15", @@ -129,51 +131,52 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.535.0", "@aws-sdk/credential-provider-node": "3.535.0", - "@getzep/zep-cloud": "1.0.6", + "@getzep/zep-cloud": "1.0.11", "@getzep/zep-js": "0.9.0", "@google-ai/generativelanguage": "2.5.0", "@google-cloud/resource-manager": "5.3.0", "@google/generative-ai": "0.11.4", "@huggingface/inference": "2.7.0", - "@langchain/anthropic": "0.1.21", + "@langchain/anthropic": "0.2.9", "@langchain/cohere": "0.0.10", - "@langchain/community": "0.2.13", - "@langchain/core": "0.2.9", - "@langchain/google-genai": "0.0.16", - "@langchain/google-vertexai": "0.0.19", - "@langchain/groq": "0.0.12", - "@langchain/mistralai": "0.0.22", - "@langchain/openai": "0.0.33", - "@langchain/pinecone": "0.0.6", + "@langchain/community": "0.2.20", + "@langchain/core": "0.2.18", + "@langchain/google-genai": "0.0.23", + "@langchain/google-vertexai": "0.0.21", + "@langchain/groq": "0.0.15", + "@langchain/mistralai": "0.0.27", + "@langchain/ollama": "^0.0.2", + "@langchain/openai": "0.2.5", + "@langchain/pinecone": "0.0.8", "@langchain/qdrant": "^0.0.5", "@langchain/redis": "0.0.5", - "@langchain/textsplitters": "0.0.2", + "@langchain/textsplitters": "0.0.3", "@mozilla/readability": "^0.5.0", "@n8n/typeorm": "0.3.20-10", - "@n8n/vm2": "3.9.20", - "@pinecone-database/pinecone": "2.2.1", + "@n8n/vm2": "3.9.25", + "@pinecone-database/pinecone": "3.0.0", "@qdrant/js-client-rest": "1.9.0", "@supabase/supabase-js": "2.43.4", - "@types/pg": "^8.11.3", + "@types/pg": "^8.11.6", "@xata.io/client": "0.28.4", - "basic-auth": "2.0.1", + "basic-auth": "catalog:", "cheerio": "1.0.0-rc.12", "cohere-ai": "7.10.1", "d3-dsv": "2.0.0", "epub2": "3.0.2", - "form-data": "4.0.0", + "form-data": "catalog:", "generate-schema": "2.6.0", "html-to-text": "9.0.5", "jsdom": "^23.0.1", "json-schema-to-zod": "2.1.0", - "langchain": "0.2.2", - "lodash": "4.17.21", + "langchain": "0.2.11", + "lodash": "catalog:", "mammoth": "1.7.2", "n8n-nodes-base": "workspace:*", "n8n-workflow": "workspace:*", - "openai": "4.47.1", + "openai": "4.53.0", "pdf-parse": "1.1.1", - "pg": "8.11.3", + "pg": "8.12.0", "redis": "4.6.12", "sqlite3": "5.1.7", "temp": "0.9.4", diff --git a/packages/@n8n/nodes-langchain/utils/N8nTool.test.ts b/packages/@n8n/nodes-langchain/utils/N8nTool.test.ts new file mode 100644 index 0000000000..6f12b18079 --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/N8nTool.test.ts @@ -0,0 +1,169 @@ +import { N8nTool } from './N8nTool'; +import { createMockExecuteFunction } from 'n8n-nodes-base/test/nodes/Helpers'; +import { z } from 'zod'; +import type { INode } from 'n8n-workflow'; +import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; + +const mockNode: INode = { + id: '1', + name: 'Mock node', + typeVersion: 2, + type: 'n8n-nodes-base.mock', + position: [60, 760], + parameters: { + operation: 'test', + }, +}; + +describe('Test N8nTool wrapper as DynamicStructuredTool', () => { + it('should wrap a tool', () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string(), + }), + }); + + expect(tool).toBeInstanceOf(DynamicStructuredTool); + }); +}); + +describe('Test N8nTool wrapper - DynamicTool fallback', () => { + it('should convert the tool to a dynamic tool', () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string(), + }), + }); + + const dynamicTool = tool.asDynamicTool(); + + expect(dynamicTool).toBeInstanceOf(DynamicTool); + }); + + it('should format fallback description correctly', () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string(), + bar: z.number().optional(), + qwe: z.boolean().describe('Boolean description'), + }), + }); + + const dynamicTool = tool.asDynamicTool(); + + expect(dynamicTool.description).toContain('foo: (description: , type: string, required: true)'); + expect(dynamicTool.description).toContain( + 'bar: (description: , type: number, required: false)', + ); + + expect(dynamicTool.description).toContain( + 'qwe: (description: Boolean description, type: boolean, required: true)', + ); + }); + + it('should handle empty parameter list correctly', () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({}), + }); + + const dynamicTool = tool.asDynamicTool(); + + expect(dynamicTool.description).toEqual('A dummy tool for testing'); + }); + + it('should parse correct parameters', async () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string().describe('Foo description'), + bar: z.number().optional(), + }), + }); + + const dynamicTool = tool.asDynamicTool(); + + const testParameters = { foo: 'some value' }; + + await dynamicTool.func(JSON.stringify(testParameters)); + + expect(func).toHaveBeenCalledWith(testParameters); + }); + + it('should recover when 1 parameter is passed directly', async () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string().describe('Foo description'), + }), + }); + + const dynamicTool = tool.asDynamicTool(); + + const testParameter = 'some value'; + + await dynamicTool.func(testParameter); + + expect(func).toHaveBeenCalledWith({ foo: testParameter }); + }); + + it('should recover when JS object is passed instead of JSON', async () => { + const func = jest.fn(); + + const ctx = createMockExecuteFunction({}, mockNode); + + const tool = new N8nTool(ctx, { + name: 'Dummy Tool', + description: 'A dummy tool for testing', + func, + schema: z.object({ + foo: z.string().describe('Foo description'), + }), + }); + + const dynamicTool = tool.asDynamicTool(); + + await dynamicTool.func('{ foo: "some value" }'); + + expect(func).toHaveBeenCalledWith({ foo: 'some value' }); + }); +}); diff --git a/packages/@n8n/nodes-langchain/utils/N8nTool.ts b/packages/@n8n/nodes-langchain/utils/N8nTool.ts new file mode 100644 index 0000000000..bb8bab08bd --- /dev/null +++ b/packages/@n8n/nodes-langchain/utils/N8nTool.ts @@ -0,0 +1,113 @@ +import type { DynamicStructuredToolInput } from '@langchain/core/tools'; +import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; +import type { IExecuteFunctions, IDataObject } from 'n8n-workflow'; +import { NodeConnectionType, jsonParse, NodeOperationError } from 'n8n-workflow'; +import { StructuredOutputParser } from 'langchain/output_parsers'; +import type { ZodTypeAny } from 'zod'; +import { ZodBoolean, ZodNullable, ZodNumber, ZodObject, ZodOptional } from 'zod'; + +const getSimplifiedType = (schema: ZodTypeAny) => { + if (schema instanceof ZodObject) { + return 'object'; + } else if (schema instanceof ZodNumber) { + return 'number'; + } else if (schema instanceof ZodBoolean) { + return 'boolean'; + } else if (schema instanceof ZodNullable || schema instanceof ZodOptional) { + return getSimplifiedType(schema.unwrap()); + } + + return 'string'; +}; + +const getParametersDescription = (parameters: Array<[string, ZodTypeAny]>) => + parameters + .map( + ([name, schema]) => + `${name}: (description: ${schema.description ?? ''}, type: ${getSimplifiedType(schema)}, required: ${!schema.isOptional()})`, + ) + .join(',\n '); + +export const prepareFallbackToolDescription = (toolDescription: string, schema: ZodObject) => { + let description = `${toolDescription}`; + + const toolParameters = Object.entries(schema.shape); + + if (toolParameters.length) { + description += ` +Tool expects valid stringified JSON object with ${toolParameters.length} properties. +Property names with description, type and required status: +${getParametersDescription(toolParameters)} +ALL parameters marked as required must be provided`; + } + + return description; +}; + +export class N8nTool extends DynamicStructuredTool { + private context: IExecuteFunctions; + + constructor(context: IExecuteFunctions, fields: DynamicStructuredToolInput) { + super(fields); + + this.context = context; + } + + asDynamicTool(): DynamicTool { + const { name, func, schema, context, description } = this; + + const parser = new StructuredOutputParser(schema); + + const wrappedFunc = async function (query: string) { + let parsedQuery: object; + + // First we try to parse the query using the structured parser (Zod schema) + try { + parsedQuery = await parser.parse(query); + } catch (e) { + // If we were unable to parse the query using the schema, we try to gracefully handle it + let dataFromModel; + + try { + // First we try to parse a JSON with more relaxed rules + dataFromModel = jsonParse(query, { acceptJSObject: true }); + } catch (error) { + // In case of error, + // If model supplied a simple string instead of an object AND only one parameter expected, we try to recover the object structure + if (Object.keys(schema.shape).length === 1) { + const parameterName = Object.keys(schema.shape)[0]; + dataFromModel = { [parameterName]: query }; + } else { + // Finally throw an error if we were unable to parse the query + throw new NodeOperationError( + context.getNode(), + `Input is not a valid JSON: ${error.message}`, + ); + } + } + + // If we were able to parse the query with a fallback, we try to validate it using the schema + // Here we will throw an error if the data still does not match the schema + parsedQuery = schema.parse(dataFromModel); + } + + try { + // Call tool function with parsed query + const result = await func(parsedQuery); + + return result; + } catch (e) { + const { index } = context.addInputData(NodeConnectionType.AiTool, [[{ json: { query } }]]); + void context.addOutputData(NodeConnectionType.AiTool, index, e); + + return e.toString(); + } + }; + + return new DynamicTool({ + name, + description: prepareFallbackToolDescription(description, schema), + func: wrappedFunc, + }); + } +} diff --git a/packages/@n8n/nodes-langchain/utils/helpers.ts b/packages/@n8n/nodes-langchain/utils/helpers.ts index c6d27ee2f6..05b4a5cadc 100644 --- a/packages/@n8n/nodes-langchain/utils/helpers.ts +++ b/packages/@n8n/nodes-langchain/utils/helpers.ts @@ -1,17 +1,18 @@ -import { NodeConnectionType, NodeOperationError, jsonStringify } from 'n8n-workflow'; import type { EventNamesAiNodesType, IDataObject, IExecuteFunctions, IWebhookFunctions, } from 'n8n-workflow'; +import { NodeConnectionType, NodeOperationError, jsonStringify } from 'n8n-workflow'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { BaseOutputParser } from '@langchain/core/output_parsers'; import type { BaseMessage } from '@langchain/core/messages'; -import { DynamicTool, type Tool } from '@langchain/core/tools'; +import type { Tool } from '@langchain/core/tools'; import type { BaseLLM } from '@langchain/core/language_models/llms'; import type { BaseChatMemory } from 'langchain/memory'; import type { BaseChatMessageHistory } from '@langchain/core/chat_history'; +import { N8nTool } from './N8nTool'; function hasMethods(obj: unknown, ...methodNames: Array): obj is T { return methodNames.every( @@ -178,7 +179,11 @@ export function serializeChatHistory(chatHistory: BaseMessage[]): string { .join('\n'); } -export const getConnectedTools = async (ctx: IExecuteFunctions, enforceUniqueNames: boolean) => { +export const getConnectedTools = async ( + ctx: IExecuteFunctions, + enforceUniqueNames: boolean, + convertStructuredTool: boolean = true, +) => { const connectedTools = ((await ctx.getInputConnectionData(NodeConnectionType.AiTool, 0)) as Tool[]) || []; @@ -186,9 +191,9 @@ export const getConnectedTools = async (ctx: IExecuteFunctions, enforceUniqueNam const seenNames = new Set(); - for (const tool of connectedTools) { - if (!(tool instanceof DynamicTool)) continue; + const finalTools = []; + for (const tool of connectedTools) { const { name } = tool; if (seenNames.has(name)) { throw new NodeOperationError( @@ -197,7 +202,13 @@ export const getConnectedTools = async (ctx: IExecuteFunctions, enforceUniqueNam ); } seenNames.add(name); + + if (convertStructuredTool && tool instanceof N8nTool) { + finalTools.push(tool.asDynamicTool()); + } else { + finalTools.push(tool); + } } - return connectedTools; + return finalTools; }; diff --git a/packages/@n8n/permissions/package.json b/packages/@n8n/permissions/package.json index e3b1c7b276..7b02f5ef02 100644 --- a/packages/@n8n/permissions/package.json +++ b/packages/@n8n/permissions/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/permissions", - "version": "0.11.0", + "version": "0.12.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/permissions/src/constants.ts b/packages/@n8n/permissions/src/constants.ts new file mode 100644 index 0000000000..de027891c5 --- /dev/null +++ b/packages/@n8n/permissions/src/constants.ts @@ -0,0 +1,23 @@ +export const DEFAULT_OPERATIONS = ['create', 'read', 'update', 'delete', 'list'] as const; +export const RESOURCES = { + auditLogs: ['manage'] as const, + banner: ['dismiss'] as const, + communityPackage: ['install', 'uninstall', 'update', 'list', 'manage'] as const, + credential: ['share', 'move', ...DEFAULT_OPERATIONS] as const, + externalSecretsProvider: ['sync', ...DEFAULT_OPERATIONS] as const, + externalSecret: ['list', 'use'] as const, + eventBusDestination: ['test', ...DEFAULT_OPERATIONS] as const, + ldap: ['sync', 'manage'] as const, + license: ['manage'] as const, + logStreaming: ['manage'] as const, + orchestration: ['read', 'list'] as const, + project: [...DEFAULT_OPERATIONS] as const, + saml: ['manage'] as const, + securityAudit: ['generate'] as const, + sourceControl: ['pull', 'push', 'manage'] as const, + tag: [...DEFAULT_OPERATIONS] as const, + user: ['resetPassword', 'changeRole', ...DEFAULT_OPERATIONS] as const, + variable: [...DEFAULT_OPERATIONS] as const, + workersView: ['manage'] as const, + workflow: ['share', 'execute', 'move', ...DEFAULT_OPERATIONS] as const, +} as const; diff --git a/packages/@n8n/permissions/src/index.ts b/packages/@n8n/permissions/src/index.ts index 0d3e510abe..f04f2e4ef6 100644 --- a/packages/@n8n/permissions/src/index.ts +++ b/packages/@n8n/permissions/src/index.ts @@ -1,3 +1,4 @@ export type * from './types'; +export * from './constants'; export * from './hasScope'; export * from './combineScopes'; diff --git a/packages/@n8n/permissions/src/types.ts b/packages/@n8n/permissions/src/types.ts index 2720272e6f..bc714734d3 100644 --- a/packages/@n8n/permissions/src/types.ts +++ b/packages/@n8n/permissions/src/types.ts @@ -1,25 +1,7 @@ -export type DefaultOperations = 'create' | 'read' | 'update' | 'delete' | 'list'; -export type Resource = - | 'auditLogs' - | 'banner' - | 'communityPackage' - | 'credential' - | 'externalSecretsProvider' - | 'externalSecret' - | 'eventBusDestination' - | 'ldap' - | 'license' - | 'logStreaming' - | 'orchestration' - | 'project' - | 'saml' - | 'securityAudit' - | 'sourceControl' - | 'tag' - | 'user' - | 'variable' - | 'workersView' - | 'workflow'; +import type { DEFAULT_OPERATIONS, RESOURCES } from './constants'; + +export type DefaultOperations = (typeof DEFAULT_OPERATIONS)[number]; +export type Resource = keyof typeof RESOURCES; export type ResourceScope< R extends Resource, diff --git a/packages/@n8n_io/eslint-config/local-rules.js b/packages/@n8n_io/eslint-config/local-rules.js index c9b533a032..92b0f6669e 100644 --- a/packages/@n8n_io/eslint-config/local-rules.js +++ b/packages/@n8n_io/eslint-config/local-rules.js @@ -182,12 +182,14 @@ module.exports = { messages: { removeSkip: 'Remove `.skip()` call', removeOnly: 'Remove `.only()` call', + removeXPrefix: 'Remove `x` prefix', }, fixable: 'code', }, create(context) { const TESTING_FUNCTIONS = new Set(['test', 'it', 'describe']); const SKIPPING_METHODS = new Set(['skip', 'only']); + const PREFIXED_TESTING_FUNCTIONS = new Set(['xtest', 'xit', 'xdescribe']); const toMessageId = (s) => 'remove' + s.charAt(0).toUpperCase() + s.slice(1); return { @@ -208,6 +210,18 @@ module.exports = { }); } }, + CallExpression(node) { + if ( + node.callee.type === 'Identifier' && + PREFIXED_TESTING_FUNCTIONS.has(node.callee.name) + ) { + context.report({ + messageId: 'removeXPrefix', + node, + fix: (fixer) => fixer.replaceText(node.callee, 'test'), + }); + } + }, }; }, }, @@ -448,6 +462,59 @@ module.exports = { }; }, }, + + 'no-type-unsafe-event-emitter': { + meta: { + type: 'problem', + docs: { + description: 'Disallow extending from `EventEmitter`, which is not type-safe.', + recommended: 'error', + }, + messages: { + noExtendsEventEmitter: 'Extend from the type-safe `TypedEmitter` class instead.', + }, + }, + create(context) { + return { + ClassDeclaration(node) { + if ( + node.superClass && + node.superClass.type === 'Identifier' && + node.superClass.name === 'EventEmitter' && + node.id.name !== 'TypedEmitter' + ) { + context.report({ + node: node.superClass, + messageId: 'noExtendsEventEmitter', + }); + } + }, + }; + }, + }, + + 'no-untyped-config-class-field': { + meta: { + type: 'problem', + docs: { + description: 'Enforce explicit typing of config class fields', + recommended: 'error', + }, + messages: { + noUntypedConfigClassField: + 'Class field must have an explicit type annotation, e.g. `field: type = value`. See: https://github.com/n8n-io/n8n/pull/10433', + }, + }, + create(context) { + return { + PropertyDefinition(node) { + if (!node.typeAnnotation) { + context.report({ node: node.key, messageId: 'noUntypedConfigClassField' }); + } + }, + }; + }, + }, }; const isJsonParseCall = (node) => diff --git a/packages/cli/.eslintrc.js b/packages/cli/.eslintrc.js index ac66574887..17ecfee499 100644 --- a/packages/cli/.eslintrc.js +++ b/packages/cli/.eslintrc.js @@ -21,6 +21,7 @@ module.exports = { rules: { 'n8n-local-rules/no-dynamic-import-template': 'error', 'n8n-local-rules/misplaced-n8n-typeorm-import': 'error', + 'n8n-local-rules/no-type-unsafe-event-emitter': 'error', complexity: 'error', // TODO: Remove this @@ -39,11 +40,17 @@ module.exports = { overrides: [ { - files: ['./src/databases/**/*.ts', './test/**/*.ts'], + files: ['./src/databases/**/*.ts', './test/**/*.ts', './src/**/__tests__/**/*.ts'], rules: { 'n8n-local-rules/misplaced-n8n-typeorm-import': 'off', }, }, + { + files: ['./test/**/*.ts', './src/**/__tests__/**/*.ts'], + rules: { + 'n8n-local-rules/no-type-unsafe-event-emitter': 'off', + }, + }, { files: ['./src/decorators/**/*.ts'], rules: { diff --git a/packages/cli/BREAKING-CHANGES.md b/packages/cli/BREAKING-CHANGES.md index 4797def422..869ace642e 100644 --- a/packages/cli/BREAKING-CHANGES.md +++ b/packages/cli/BREAKING-CHANGES.md @@ -2,6 +2,16 @@ This list shows all the versions which include breaking changes and how to upgrade. +## 1.55.0 + +### What changed? + +The `N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES` environment variable now also blocks access to n8n's static cache directory at `~/.cache/n8n/public`. + +### When is action necessary? + +If you are writing to or reading from a file at n8n's static cache directory via a node, e.g. `Read/Write Files from Disk`, please update your node to use a different path. + ## 1.52.0 ### What changed? diff --git a/packages/cli/README.md b/packages/cli/README.md index beeac69369..526f463167 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -147,8 +147,4 @@ You can also find breaking changes here: [Breaking Changes](./BREAKING-CHANGES.m ## License -n8n is [fair-code](https://faircode.io) distributed under the [**Sustainable Use License**](https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md). - -Proprietary licenses are available for enterprise customers. [Get in touch](mailto:license@n8n.io) - -Additional information about the license can be found in the [docs](https://docs.n8n.io/reference/license/). +You can find the license information [here](https://github.com/n8n-io/n8n/blob/master/README.md#license) diff --git a/packages/cli/package.json b/packages/cli/package.json index e69fb4fc26..c7608d1c34 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "1.52.0", + "version": "1.56.0", "description": "n8n Workflow Automation Tool", "main": "dist/index", "types": "dist/index.d.ts", @@ -56,12 +56,12 @@ "@types/compression": "1.0.1", "@types/convict": "^6.1.1", "@types/cookie-parser": "^1.4.2", - "@types/express": "^4.17.21", + "@types/express": "catalog:", "@types/flat": "^5.0.5", "@types/formidable": "^3.4.5", "@types/json-diff": "^1.0.0", "@types/jsonwebtoken": "^9.0.6", - "@types/lodash": "^4.14.195", + "@types/lodash": "catalog:", "@types/psl": "^1.1.0", "@types/replacestream": "^4.0.1", "@types/shelljs": "^0.8.11", @@ -69,10 +69,10 @@ "@types/superagent": "^8.1.7", "@types/swagger-ui-express": "^4.1.6", "@types/syslog-client": "^1.1.2", - "@types/uuid": "^8.3.2", + "@types/uuid": "catalog:", "@types/validator": "^13.7.0", "@types/ws": "^8.5.4", - "@types/xml2js": "^0.4.14", + "@types/xml2js": "catalog:", "@types/yamljs": "^0.2.31", "@vvo/tzdb": "^6.141.0", "chokidar": "^3.5.2", @@ -83,19 +83,21 @@ "dependencies": { "@azure/identity": "^4.3.0", "@azure/keyvault-secrets": "^4.8.0", + "@google-cloud/secret-manager": "^5.6.0", "@n8n/client-oauth2": "workspace:*", "@n8n/config": "workspace:*", - "@n8n/localtunnel": "2.1.0", + "@n8n/localtunnel": "3.0.0", "@n8n/n8n-nodes-langchain": "workspace:*", "@n8n/permissions": "workspace:*", "@n8n/typeorm": "0.3.20-10", - "@n8n_io/license-sdk": "2.13.0", + "@n8n_io/ai-assistant-sdk": "1.9.4", + "@n8n_io/license-sdk": "2.13.1", "@oclif/core": "4.0.7", - "@rudderstack/rudder-sdk-node": "2.0.7", + "@rudderstack/rudder-sdk-node": "2.0.9", "@sentry/integrations": "7.87.0", "@sentry/node": "7.87.0", "aws4": "1.11.0", - "axios": "1.6.7", + "axios": "catalog:", "bcryptjs": "2.4.3", "bull": "4.12.1", "cache-manager": "5.2.3", @@ -112,10 +114,10 @@ "express": "4.19.2", "express-async-errors": "3.1.1", "express-handlebars": "7.1.2", - "express-openapi-validator": "5.1.6", + "express-openapi-validator": "5.3.1", "express-prom-bundle": "6.6.0", "express-rate-limit": "7.2.0", - "fast-glob": "3.2.12", + "fast-glob": "catalog:", "flat": "5.0.2", "flatted": "3.2.7", "formidable": "3.5.1", @@ -128,14 +130,14 @@ "jsonschema": "1.4.1", "jsonwebtoken": "9.0.2", "ldapts": "4.2.6", - "lodash": "4.17.21", - "luxon": "3.3.0", - "mysql2": "3.10.0", + "lodash": "catalog:", + "luxon": "catalog:", + "mysql2": "3.11.0", "n8n-core": "workspace:*", "n8n-editor-ui": "workspace:*", "n8n-nodes-base": "workspace:*", "n8n-workflow": "workspace:*", - "nanoid": "3.3.6", + "nanoid": "catalog:", "nodemailer": "6.9.9", "oauth-1.0a": "2.2.6", "open": "7.4.2", @@ -143,8 +145,8 @@ "otpauth": "9.1.1", "p-cancelable": "2.1.1", "p-lazy": "3.1.0", - "pg": "8.11.3", - "picocolors": "1.0.0", + "pg": "8.12.0", + "picocolors": "1.0.1", "pkce-challenge": "3.0.0", "posthog-node": "3.2.1", "prom-client": "13.2.0", @@ -162,13 +164,14 @@ "sshpk": "1.17.0", "swagger-ui-express": "5.0.0", "syslog-client": "1.1.1", - "typedi": "0.10.0", - "uuid": "8.3.2", + "typedi": "catalog:", + "uuid": "catalog:", "validator": "13.7.0", "winston": "3.8.2", "ws": "8.17.1", - "xml2js": "0.6.2", + "xml2js": "catalog:", "xmllint-wasm": "3.0.1", + "xss": "^1.0.14", "yamljs": "0.3.0", "zod": "3.22.4" } diff --git a/packages/cli/scripts/build.mjs b/packages/cli/scripts/build.mjs index b1f5f5bd8f..36076ccfd4 100644 --- a/packages/cli/scripts/build.mjs +++ b/packages/cli/scripts/build.mjs @@ -23,8 +23,8 @@ if (publicApiEnabled) { function copyUserManagementEmailTemplates() { const templates = { - source: path.resolve(ROOT_DIR, 'src', 'UserManagement', 'email', 'templates'), - destination: path.resolve(ROOT_DIR, 'dist', 'UserManagement', 'email'), + source: path.resolve(ROOT_DIR, 'src', 'user-management', 'email', 'templates'), + destination: path.resolve(ROOT_DIR, 'dist', 'user-management', 'email'), }; shell.cp('-r', templates.source, templates.destination); diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 25f758185a..f6c3f4670f 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -1,4 +1,4 @@ -import type { Application, Request, Response } from 'express'; +import type { Application } from 'express'; import type { ExecutionError, ICredentialDataDecryptedObject, @@ -22,11 +22,10 @@ import type { FeatureFlags, INodeProperties, IUserSettings, - IHttpRequestMethods, StartNodeData, } from 'n8n-workflow'; -import type { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import type { ActiveWorkflowManager } from '@/active-workflow-manager'; import type { WorkflowExecute } from 'n8n-core'; @@ -40,10 +39,10 @@ import type { CredentialsRepository } from '@db/repositories/credentials.reposit import type { SettingsRepository } from '@db/repositories/settings.repository'; import type { UserRepository } from '@db/repositories/user.repository'; import type { WorkflowRepository } from '@db/repositories/workflow.repository'; -import type { ExternalHooks } from './ExternalHooks'; +import type { ExternalHooks } from './external-hooks'; import type { LICENSE_FEATURES, LICENSE_QUOTAS } from './constants'; import type { WorkflowWithSharingsAndCredentials } from './workflows/workflows.types'; -import type { WorkerJobStatusSummary } from './services/orchestration/worker/types'; +import type { RunningJobSummary } from './scaling/types'; import type { Scope } from '@n8n/permissions'; export interface ICredentialsTypeData { @@ -239,42 +238,6 @@ export interface IExternalHooksFunctions { }; } -export type WebhookCORSRequest = Request & { method: 'OPTIONS' }; - -export type WebhookRequest = Request<{ path: string }> & { - method: IHttpRequestMethods; - params: Record; -}; - -export type WaitingWebhookRequest = WebhookRequest & { - params: WebhookRequest['path'] & { suffix?: string }; -}; - -export interface WebhookAccessControlOptions { - allowedOrigins?: string; -} - -export interface IWebhookManager { - /** Gets all request methods associated with a webhook path*/ - getWebhookMethods?: (path: string) => Promise; - - /** Find the CORS options matching a path and method */ - findAccessControlOptions?: ( - path: string, - httpMethod: IHttpRequestMethods, - ) => Promise; - - executeWebhook(req: WebhookRequest, res: Response): Promise; -} - -export interface ITelemetryUserDeletionData { - user_id: string; - target_user_old_status: 'active' | 'invited'; - migration_strategy?: 'transfer_data' | 'delete_data'; - target_user_id?: string; - migration_user_id?: string; -} - export interface IVersionNotificationSettings { enabled: boolean; endpoint: string; @@ -457,7 +420,7 @@ export interface IPushDataWorkerStatusMessage { export interface IPushDataWorkerStatusPayload { workerId: string; - runningJobsSummary: WorkerJobStatusSummary[]; + runningJobsSummary: RunningJobSummary[]; freeMem: number; totalMem: number; uptime: number; @@ -474,13 +437,6 @@ export interface IPushDataWorkerStatusPayload { version: string; } -export interface IResponseCallbackData { - data?: IDataObject | IDataObject[]; - headers?: object; - noWebhookResponse?: boolean; - responseCode?: number; -} - export interface INodesTypeData { [key: string]: { className: string; diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts deleted file mode 100644 index 9f02fd67ef..0000000000 --- a/packages/cli/src/InternalHooks.ts +++ /dev/null @@ -1,641 +0,0 @@ -import { Service } from 'typedi'; -import { snakeCase } from 'change-case'; -import os from 'node:os'; -import { get as pslGet } from 'psl'; -import { GlobalConfig } from '@n8n/config'; -import type { - ExecutionStatus, - INodesGraphResult, - IRun, - ITelemetryTrackProperties, - IWorkflowBase, -} from 'n8n-workflow'; -import { TelemetryHelpers } from 'n8n-workflow'; -import { InstanceSettings } from 'n8n-core'; - -import config from '@/config'; -import { N8N_VERSION } from '@/constants'; -import type { AuthProviderType } from '@db/entities/AuthIdentity'; -import type { User } from '@db/entities/User'; -import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; -import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { determineFinalExecutionStatus } from '@/executionLifecycleHooks/shared/sharedHookFunctions'; -import type { - ITelemetryUserDeletionData, - IWorkflowDb, - IExecutionTrackProperties, -} from '@/Interfaces'; -import { License } from '@/License'; -import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; -import { NodeTypes } from '@/NodeTypes'; -import { Telemetry } from '@/telemetry'; -import type { Project } from '@db/entities/Project'; -import { ProjectRelationRepository } from './databases/repositories/projectRelation.repository'; -import { SharedCredentialsRepository } from './databases/repositories/sharedCredentials.repository'; -import { MessageEventBus } from './eventbus/MessageEventBus/MessageEventBus'; - -/** - * @deprecated Do not add to this class. To add audit or telemetry events, use - * `EventService` to emit the event and then use the `AuditEventRelay` or - * `TelemetryEventRelay` to forward them to the event bus or telemetry. - */ -@Service() -export class InternalHooks { - constructor( - private readonly globalConfig: GlobalConfig, - private readonly telemetry: Telemetry, - private readonly nodeTypes: NodeTypes, - private readonly sharedWorkflowRepository: SharedWorkflowRepository, - private readonly workflowRepository: WorkflowRepository, - workflowStatisticsService: WorkflowStatisticsService, - private readonly instanceSettings: InstanceSettings, - private readonly license: License, - private readonly projectRelationRepository: ProjectRelationRepository, - private readonly sharedCredentialsRepository: SharedCredentialsRepository, - private readonly _eventBus: MessageEventBus, // needed until we decouple telemetry - ) { - workflowStatisticsService.on( - 'telemetry.onFirstProductionWorkflowSuccess', - async (metrics) => await this.onFirstProductionWorkflowSuccess(metrics), - ); - workflowStatisticsService.on( - 'telemetry.onFirstWorkflowDataLoad', - async (metrics) => await this.onFirstWorkflowDataLoad(metrics), - ); - } - - async init() { - await this.telemetry.init(); - } - - async onServerStarted(): Promise { - const cpus = os.cpus(); - const binaryDataConfig = config.getEnv('binaryDataManager'); - - const isS3Selected = config.getEnv('binaryDataManager.mode') === 's3'; - const isS3Available = config.getEnv('binaryDataManager.availableModes').includes('s3'); - const isS3Licensed = this.license.isBinaryDataS3Licensed(); - const authenticationMethod = config.getEnv('userManagement.authenticationMethod'); - - const info = { - version_cli: N8N_VERSION, - db_type: this.globalConfig.database.type, - n8n_version_notifications_enabled: this.globalConfig.versionNotifications.enabled, - n8n_disable_production_main_process: config.getEnv( - 'endpoints.disableProductionWebhooksOnMainProcess', - ), - system_info: { - os: { - type: os.type(), - version: os.version(), - }, - memory: os.totalmem() / 1024, - cpus: { - count: cpus.length, - model: cpus[0].model, - speed: cpus[0].speed, - }, - }, - execution_variables: { - executions_mode: config.getEnv('executions.mode'), - executions_timeout: config.getEnv('executions.timeout'), - executions_timeout_max: config.getEnv('executions.maxTimeout'), - executions_data_save_on_error: config.getEnv('executions.saveDataOnError'), - executions_data_save_on_success: config.getEnv('executions.saveDataOnSuccess'), - executions_data_save_on_progress: config.getEnv('executions.saveExecutionProgress'), - executions_data_save_manual_executions: config.getEnv( - 'executions.saveDataManualExecutions', - ), - executions_data_prune: config.getEnv('executions.pruneData'), - executions_data_max_age: config.getEnv('executions.pruneDataMaxAge'), - }, - n8n_deployment_type: config.getEnv('deployment.type'), - n8n_binary_data_mode: binaryDataConfig.mode, - smtp_set_up: this.globalConfig.userManagement.emails.mode === 'smtp', - ldap_allowed: authenticationMethod === 'ldap', - saml_enabled: authenticationMethod === 'saml', - license_plan_name: this.license.getPlanName(), - license_tenant_id: config.getEnv('license.tenantId'), - binary_data_s3: isS3Available && isS3Selected && isS3Licensed, - multi_main_setup_enabled: config.getEnv('multiMainSetup.enabled'), - }; - - const firstWorkflow = await this.workflowRepository.findOne({ - select: ['createdAt'], - order: { createdAt: 'ASC' }, - where: {}, - }); - - return await Promise.all([ - this.telemetry.identify(info), - this.telemetry.track('Instance started', { - ...info, - earliest_workflow_created: firstWorkflow?.createdAt, - }), - ]); - } - - async onFrontendSettingsAPI(pushRef?: string): Promise { - return await this.telemetry.track('Session started', { session_id: pushRef }); - } - - async onPersonalizationSurveySubmitted( - userId: string, - answers: Record, - ): Promise { - const camelCaseKeys = Object.keys(answers); - const personalizationSurveyData = { user_id: userId } as Record; - camelCaseKeys.forEach((camelCaseKey) => { - personalizationSurveyData[snakeCase(camelCaseKey)] = answers[camelCaseKey]; - }); - - return await this.telemetry.track( - 'User responded to personalization questions', - personalizationSurveyData, - ); - } - - async onWorkflowCreated( - user: User, - workflow: IWorkflowBase, - project: Project, - publicApi: boolean, - ): Promise { - const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); - - void this.telemetry.track('User created workflow', { - user_id: user.id, - workflow_id: workflow.id, - node_graph_string: JSON.stringify(nodeGraph), - public_api: publicApi, - project_id: project.id, - project_type: project.type, - }); - } - - async onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise { - void this.telemetry.track('User deleted workflow', { - user_id: user.id, - workflow_id: workflowId, - public_api: publicApi, - }); - } - - async onWorkflowSaved(user: User, workflow: IWorkflowDb, publicApi: boolean): Promise { - const isCloudDeployment = config.getEnv('deployment.type') === 'cloud'; - - const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, { - isCloudDeployment, - }); - - let userRole: 'owner' | 'sharee' | 'member' | undefined = undefined; - const role = await this.sharedWorkflowRepository.findSharingRole(user.id, workflow.id); - if (role) { - userRole = role === 'workflow:owner' ? 'owner' : 'sharee'; - } else { - const workflowOwner = await this.sharedWorkflowRepository.getWorkflowOwningProject( - workflow.id, - ); - - if (workflowOwner) { - const projectRole = await this.projectRelationRepository.findProjectRole({ - userId: user.id, - projectId: workflowOwner.id, - }); - - if (projectRole && projectRole !== 'project:personalOwner') { - userRole = 'member'; - } - } - } - - const notesCount = Object.keys(nodeGraph.notes).length; - const overlappingCount = Object.values(nodeGraph.notes).filter( - (note) => note.overlapping, - ).length; - - void this.telemetry.track('User saved workflow', { - user_id: user.id, - workflow_id: workflow.id, - node_graph_string: JSON.stringify(nodeGraph), - notes_count_overlapping: overlappingCount, - notes_count_non_overlapping: notesCount - overlappingCount, - version_cli: N8N_VERSION, - num_tags: workflow.tags?.length ?? 0, - public_api: publicApi, - sharing_role: userRole, - }); - } - - // eslint-disable-next-line complexity - async onWorkflowPostExecute( - _executionId: string, - workflow: IWorkflowBase, - runData?: IRun, - userId?: string, - ): Promise { - if (!workflow.id) { - return; - } - - if (runData?.status === 'waiting') { - // No need to send telemetry or logs when the workflow hasn't finished yet. - return; - } - - const promises = []; - - const telemetryProperties: IExecutionTrackProperties = { - workflow_id: workflow.id, - is_manual: false, - version_cli: N8N_VERSION, - success: false, - }; - - if (userId) { - telemetryProperties.user_id = userId; - } - - if (runData?.data.resultData.error?.message?.includes('canceled')) { - runData.status = 'canceled'; - } - - telemetryProperties.success = !!runData?.finished; - - // const executionStatus: ExecutionStatus = runData?.status ?? 'unknown'; - const executionStatus: ExecutionStatus = runData - ? determineFinalExecutionStatus(runData) - : 'unknown'; - - if (runData !== undefined) { - telemetryProperties.execution_mode = runData.mode; - telemetryProperties.is_manual = runData.mode === 'manual'; - - let nodeGraphResult: INodesGraphResult | null = null; - - if (!telemetryProperties.success && runData?.data.resultData.error) { - telemetryProperties.error_message = runData?.data.resultData.error.message; - let errorNodeName = - 'node' in runData?.data.resultData.error - ? runData?.data.resultData.error.node?.name - : undefined; - telemetryProperties.error_node_type = - 'node' in runData?.data.resultData.error - ? runData?.data.resultData.error.node?.type - : undefined; - - if (runData.data.resultData.lastNodeExecuted) { - const lastNode = TelemetryHelpers.getNodeTypeForName( - workflow, - runData.data.resultData.lastNodeExecuted, - ); - - if (lastNode !== undefined) { - telemetryProperties.error_node_type = lastNode.type; - errorNodeName = lastNode.name; - } - } - - if (telemetryProperties.is_manual) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); - telemetryProperties.node_graph = nodeGraphResult.nodeGraph; - telemetryProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); - - if (errorNodeName) { - telemetryProperties.error_node_id = nodeGraphResult.nameIndices[errorNodeName]; - } - } - } - - if (telemetryProperties.is_manual) { - if (!nodeGraphResult) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); - } - - let userRole: 'owner' | 'sharee' | undefined = undefined; - if (userId) { - const role = await this.sharedWorkflowRepository.findSharingRole(userId, workflow.id); - if (role) { - userRole = role === 'workflow:owner' ? 'owner' : 'sharee'; - } - } - - const manualExecEventProperties: ITelemetryTrackProperties = { - user_id: userId, - workflow_id: workflow.id, - status: executionStatus, - executionStatus: runData?.status ?? 'unknown', - error_message: telemetryProperties.error_message as string, - error_node_type: telemetryProperties.error_node_type, - node_graph_string: telemetryProperties.node_graph_string as string, - error_node_id: telemetryProperties.error_node_id as string, - webhook_domain: null, - sharing_role: userRole, - }; - - if (!manualExecEventProperties.node_graph_string) { - nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); - manualExecEventProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); - } - - if (runData.data.startData?.destinationNode) { - const telemetryPayload = { - ...manualExecEventProperties, - node_type: TelemetryHelpers.getNodeTypeForName( - workflow, - runData.data.startData?.destinationNode, - )?.type, - node_id: nodeGraphResult.nameIndices[runData.data.startData?.destinationNode], - }; - - promises.push(this.telemetry.track('Manual node exec finished', telemetryPayload)); - } else { - nodeGraphResult.webhookNodeNames.forEach((name: string) => { - const execJson = runData.data.resultData.runData[name]?.[0]?.data?.main?.[0]?.[0] - ?.json as { headers?: { origin?: string } }; - if (execJson?.headers?.origin && execJson.headers.origin !== '') { - manualExecEventProperties.webhook_domain = pslGet( - execJson.headers.origin.replace(/^https?:\/\//, ''), - ); - } - }); - - promises.push( - this.telemetry.track('Manual workflow exec finished', manualExecEventProperties), - ); - } - } - } - - void Promise.all([...promises, this.telemetry.trackWorkflowExecution(telemetryProperties)]); - } - - async onWorkflowSharingUpdate(workflowId: string, userId: string, userList: string[]) { - const properties: ITelemetryTrackProperties = { - workflow_id: workflowId, - user_id_sharer: userId, - user_id_list: userList, - }; - - return await this.telemetry.track('User updated workflow sharing', properties); - } - - async onN8nStop(): Promise { - const timeoutPromise = new Promise((resolve) => { - setTimeout(() => { - resolve(); - }, 3000); - }); - - return await Promise.race([timeoutPromise, this.telemetry.trackN8nStop()]); - } - - async onUserDeletion(userDeletionData: { - user: User; - telemetryData: ITelemetryUserDeletionData; - publicApi: boolean; - }): Promise { - void this.telemetry.track('User deleted user', { - ...userDeletionData.telemetryData, - user_id: userDeletionData.user.id, - public_api: userDeletionData.publicApi, - }); - } - - async onUserInvite(userInviteData: { - user: User; - target_user_id: string[]; - public_api: boolean; - email_sent: boolean; - invitee_role: string; - }): Promise { - void this.telemetry.track('User invited new user', { - user_id: userInviteData.user.id, - target_user_id: userInviteData.target_user_id, - public_api: userInviteData.public_api, - email_sent: userInviteData.email_sent, - invitee_role: userInviteData.invitee_role, - }); - } - - async onUserRoleChange(userRoleChangeData: { - user: User; - target_user_id: string; - public_api: boolean; - target_user_new_role: string; - }) { - const { user, ...rest } = userRoleChangeData; - - void this.telemetry.track('User changed role', { user_id: user.id, ...rest }); - } - - async onUserRetrievedUser(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved user', userRetrievedData); - } - - async onUserRetrievedAllUsers(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved all users', userRetrievedData); - } - - async onUserRetrievedExecution(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved execution', userRetrievedData); - } - - async onUserRetrievedAllExecutions(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved all executions', userRetrievedData); - } - - async onUserRetrievedWorkflow(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved workflow', userRetrievedData); - } - - async onUserRetrievedAllWorkflows(userRetrievedData: { - user_id: string; - public_api: boolean; - }): Promise { - return await this.telemetry.track('User retrieved all workflows', userRetrievedData); - } - - async onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise { - void this.telemetry.track('User changed personal settings', { - user_id: userUpdateData.user.id, - fields_changed: userUpdateData.fields_changed, - }); - } - - async onUserInviteEmailClick(userInviteClickData: { - inviter: User; - invitee: User; - }): Promise { - void this.telemetry.track('User clicked invite link from email', { - user_id: userInviteClickData.invitee.id, - }); - } - - async onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise { - void this.telemetry.track('User clicked password reset link from email', { - user_id: userPasswordResetData.user.id, - }); - } - - async onUserTransactionalEmail(userTransactionalEmailData: { - user_id: string; - message_type: - | 'Reset password' - | 'New user invite' - | 'Resend invite' - | 'Workflow shared' - | 'Credentials shared'; - public_api: boolean; - }): Promise { - return await this.telemetry.track( - 'Instance sent transactional email to user', - userTransactionalEmailData, - ); - } - - async onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise { - void this.telemetry.track('User requested password reset while logged out', { - user_id: userPasswordResetData.user.id, - }); - } - - async onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise { - return await this.telemetry.track('Owner finished instance setup', instanceOwnerSetupData); - } - - async onUserSignup( - user: User, - userSignupData: { - user_type: AuthProviderType; - was_disabled_ldap_user: boolean; - }, - ): Promise { - void this.telemetry.track('User signed up', { - user_id: user.id, - ...userSignupData, - }); - } - - async onEmailFailed(failedEmailData: { - user: User; - message_type: - | 'Reset password' - | 'New user invite' - | 'Resend invite' - | 'Workflow shared' - | 'Credentials shared'; - public_api: boolean; - }): Promise { - void this.telemetry.track('Instance failed to send transactional email to user', { - user_id: failedEmailData.user.id, - }); - } - - /** - * Credentials - */ - - async onUserCreatedCredentials(userCreatedCredentialsData: { - user: User; - credential_name: string; - credential_type: string; - credential_id: string; - public_api: boolean; - }): Promise { - const project = await this.sharedCredentialsRepository.findCredentialOwningProject( - userCreatedCredentialsData.credential_id, - ); - void this.telemetry.track('User created credentials', { - user_id: userCreatedCredentialsData.user.id, - credential_type: userCreatedCredentialsData.credential_type, - credential_id: userCreatedCredentialsData.credential_id, - instance_id: this.instanceSettings.instanceId, - project_id: project?.id, - project_type: project?.type, - }); - } - - async onUserSharedCredentials(userSharedCredentialsData: { - user: User; - credential_name: string; - credential_type: string; - credential_id: string; - user_id_sharer: string; - user_ids_sharees_added: string[]; - sharees_removed: number | null; - }): Promise { - void this.telemetry.track('User updated cred sharing', { - user_id: userSharedCredentialsData.user.id, - credential_type: userSharedCredentialsData.credential_type, - credential_id: userSharedCredentialsData.credential_id, - user_id_sharer: userSharedCredentialsData.user_id_sharer, - user_ids_sharees_added: userSharedCredentialsData.user_ids_sharees_added, - sharees_removed: userSharedCredentialsData.sharees_removed, - instance_id: this.instanceSettings.instanceId, - }); - } - - async onUserUpdatedCredentials(userUpdatedCredentialsData: { - user: User; - credential_name: string; - credential_type: string; - credential_id: string; - }): Promise { - void this.telemetry.track('User updated credentials', { - user_id: userUpdatedCredentialsData.user.id, - credential_type: userUpdatedCredentialsData.credential_type, - credential_id: userUpdatedCredentialsData.credential_id, - }); - } - - async onUserDeletedCredentials(userUpdatedCredentialsData: { - user: User; - credential_name: string; - credential_type: string; - credential_id: string; - }): Promise { - void this.telemetry.track('User deleted credentials', { - user_id: userUpdatedCredentialsData.user.id, - credential_type: userUpdatedCredentialsData.credential_type, - credential_id: userUpdatedCredentialsData.credential_id, - instance_id: this.instanceSettings.instanceId, - }); - } - - /* - * Execution Statistics - */ - async onFirstProductionWorkflowSuccess(data: { - user_id: string; - workflow_id: string; - }): Promise { - return await this.telemetry.track('Workflow first prod success', data); - } - - async onFirstWorkflowDataLoad(data: { - user_id: string; - workflow_id: string; - node_type: string; - node_id: string; - credential_type?: string; - credential_id?: string; - }): Promise { - return await this.telemetry.track('Workflow first data fetched', data); - } -} diff --git a/packages/cli/src/PublicApi/index.ts b/packages/cli/src/PublicApi/index.ts index 3ebb67f210..35c6d9862e 100644 --- a/packages/cli/src/PublicApi/index.ts +++ b/packages/cli/src/PublicApi/index.ts @@ -10,14 +10,12 @@ import type { HttpError } from 'express-openapi-validator/dist/framework/types'; import type { OpenAPIV3 } from 'openapi-types'; import type { JsonObject } from 'swagger-ui-express'; -import config from '@/config'; - -import { License } from '@/License'; +import { License } from '@/license'; import { UserRepository } from '@db/repositories/user.repository'; import { UrlService } from '@/services/url.service'; import type { AuthenticatedRequest } from '@/requests'; import { GlobalConfig } from '@n8n/config'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; async function createApiRouter( version: string, @@ -25,7 +23,7 @@ async function createApiRouter( handlersDirectory: string, publicApiEndpoint: string, ): Promise { - const n8nPath = config.getEnv('path'); + const n8nPath = Container.get(GlobalConfig).path; const swaggerDocument = YAML.load(openApiSpecPath) as JsonObject; // add the server depending on the config so the user can interact with the API // from the Swagger UI diff --git a/packages/cli/src/PublicApi/types.ts b/packages/cli/src/PublicApi/types.ts index 2e80deb1af..631a8c09d7 100644 --- a/packages/cli/src/PublicApi/types.ts +++ b/packages/cli/src/PublicApi/types.ts @@ -29,6 +29,7 @@ export declare namespace ExecutionRequest { includeData?: boolean; workflowId?: string; lastId?: string; + projectId?: string; } >; @@ -72,6 +73,7 @@ export declare namespace WorkflowRequest { workflowId?: number; active: boolean; name?: string; + projectId?: string; } >; @@ -82,6 +84,11 @@ export declare namespace WorkflowRequest { type Activate = Get; type GetTags = Get; type UpdateTags = AuthenticatedRequest<{ id: string }, {}, TagEntity[]>; + type Transfer = AuthenticatedRequest< + { workflowId: string }, + {}, + { destinationProjectId: string } + >; } export declare namespace UserRequest { @@ -136,6 +143,12 @@ export declare namespace CredentialRequest { >; type Delete = AuthenticatedRequest<{ id: string }, {}, {}, Record>; + + type Transfer = AuthenticatedRequest< + { workflowId: string }, + {}, + { destinationProjectId: string } + >; } export type OperationID = 'getUsers' | 'getUser'; diff --git a/packages/cli/src/PublicApi/v1/handlers/audit/audit.handler.ts b/packages/cli/src/PublicApi/v1/handlers/audit/audit.handler.ts index caf3750ad4..8aa09cb902 100644 --- a/packages/cli/src/PublicApi/v1/handlers/audit/audit.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/audit/audit.handler.ts @@ -8,7 +8,7 @@ export = { globalScope('securityAudit:generate'), async (req: AuditRequest.Generate, res: Response): Promise => { try { - const { SecurityAuditService } = await import('@/security-audit/SecurityAudit.service'); + const { SecurityAuditService } = await import('@/security-audit/security-audit.service'); const result = await Container.get(SecurityAuditService).run( req.body?.additionalOptions?.categories, req.body?.additionalOptions?.daysAbandonedWorkflow, diff --git a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.handler.ts b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.handler.ts index 4da7635831..2c13f6de56 100644 --- a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.handler.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-argument */ import type express from 'express'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import { CredentialTypes } from '@/CredentialTypes'; +import { CredentialsHelper } from '@/credentials-helper'; +import { CredentialTypes } from '@/credential-types'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; import type { CredentialTypeRequest, CredentialRequest } from '../../../types'; import { projectScope } from '../../shared/middlewares/global.middleware'; @@ -19,6 +19,8 @@ import { toJsonSchema, } from './credentials.service'; import { Container } from 'typedi'; +import { z } from 'zod'; +import { EnterpriseCredentialsService } from '@/credentials/credentials.service.ee'; export = { createCredential: [ @@ -44,6 +46,20 @@ export = { } }, ], + transferCredential: [ + projectScope('credential:move', 'credential'), + async (req: CredentialRequest.Transfer, res: express.Response) => { + const body = z.object({ destinationProjectId: z.string() }).parse(req.body); + + await Container.get(EnterpriseCredentialsService).transferOne( + req.user, + req.params.workflowId, + body.destinationProjectId, + ); + + res.status(204).send(); + }, + ], deleteCredential: [ projectScope('credential:delete', 'credential'), async ( diff --git a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.middleware.ts b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.middleware.ts index b7c2412923..8583c866b8 100644 --- a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.middleware.ts +++ b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.middleware.ts @@ -3,8 +3,8 @@ import type express from 'express'; import { validate } from 'jsonschema'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import { CredentialTypes } from '@/CredentialTypes'; +import { CredentialsHelper } from '@/credentials-helper'; +import { CredentialTypes } from '@/credential-types'; import type { CredentialRequest } from '../../../types'; import { toJsonSchema } from './credentials.service'; import { Container } from 'typedi'; diff --git a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.service.ts b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.service.ts index 524dc608db..075c7fcf11 100644 --- a/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.service.ts +++ b/packages/cli/src/PublicApi/v1/handlers/credentials/credentials.service.ts @@ -10,15 +10,14 @@ import type { ICredentialsDb } from '@/Interfaces'; import { CredentialsEntity } from '@db/entities/CredentialsEntity'; import { SharedCredentials } from '@db/entities/SharedCredentials'; import type { User } from '@db/entities/User'; -import { ExternalHooks } from '@/ExternalHooks'; +import { ExternalHooks } from '@/external-hooks'; import type { IDependency, IJsonSchema } from '../../../types'; import type { CredentialRequest } from '@/requests'; import { Container } from 'typedi'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; -import { InternalHooks } from '@/InternalHooks'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; export async function getCredentials(credentialId: string): Promise { return await Container.get(CredentialsRepository).findOneBy({ id: credentialId }); @@ -52,22 +51,7 @@ export async function saveCredential( user: User, encryptedData: ICredentialsDb, ): Promise { - await Container.get(ExternalHooks).run('credentials.create', [encryptedData]); - void Container.get(InternalHooks).onUserCreatedCredentials({ - user, - credential_name: credential.name, - credential_type: credential.type, - credential_id: credential.id, - public_api: true, - }); - Container.get(EventService).emit('credentials-created', { - user, - credentialName: credential.name, - credentialType: credential.type, - credentialId: credential.id, - }); - - return await Db.transaction(async (transactionManager) => { + const result = await Db.transaction(async (transactionManager) => { const savedCredential = await transactionManager.save(credential); savedCredential.data = credential.data; @@ -89,6 +73,23 @@ export async function saveCredential( return savedCredential; }); + + await Container.get(ExternalHooks).run('credentials.create', [encryptedData]); + + const project = await Container.get(SharedCredentialsRepository).findCredentialOwningProject( + credential.id, + ); + + Container.get(EventService).emit('credentials-created', { + user, + credentialType: credential.type, + credentialId: credential.id, + projectId: project?.id, + projectType: project?.type, + publicApi: true, + }); + + return result; } export async function removeCredential( @@ -96,15 +97,8 @@ export async function removeCredential( credentials: CredentialsEntity, ): Promise { await Container.get(ExternalHooks).run('credentials.delete', [credentials.id]); - void Container.get(InternalHooks).onUserDeletedCredentials({ - user, - credential_name: credentials.name, - credential_type: credentials.type, - credential_id: credentials.id, - }); Container.get(EventService).emit('credentials-deleted', { user, - credentialName: credentials.name, credentialType: credentials.type, credentialId: credentials.id, }); diff --git a/packages/cli/src/PublicApi/v1/handlers/credentials/spec/paths/credentials.id.transfer.yml b/packages/cli/src/PublicApi/v1/handlers/credentials/spec/paths/credentials.id.transfer.yml new file mode 100644 index 0000000000..a9e9c5cf7c --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/credentials/spec/paths/credentials.id.transfer.yml @@ -0,0 +1,31 @@ +put: + x-eov-operation-id: transferCredential + x-eov-operation-handler: v1/handlers/credentials/credentials.handler + tags: + - Workflow + summary: Transfer a credential to another project. + description: Transfer a credential to another project. + parameters: + - $ref: '../schemas/parameters/credentialId.yml' + requestBody: + description: Destination project for the credential transfer. + content: + application/json: + schema: + type: object + properties: + destinationProjectId: + type: string + description: The ID of the project to transfer the credential to. + required: + - destinationProjectId + required: true + responses: + '200': + description: Operation successful. + '400': + $ref: '../../../../shared/spec/responses/badRequest.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/credentials/spec/schemas/parameters/credentialId.yml b/packages/cli/src/PublicApi/v1/handlers/credentials/spec/schemas/parameters/credentialId.yml new file mode 100644 index 0000000000..f16676ce0b --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/credentials/spec/schemas/parameters/credentialId.yml @@ -0,0 +1,6 @@ +name: id +in: path +description: The ID of the credential. +required: true +schema: + type: string diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts index cd936d1d1c..beba4606aa 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts @@ -2,12 +2,12 @@ import type express from 'express'; import { Container } from 'typedi'; import { replaceCircularReferences } from 'n8n-workflow'; -import { ActiveExecutions } from '@/ActiveExecutions'; +import { ActiveExecutions } from '@/active-executions'; import { validCursor } from '../../shared/middlewares/global.middleware'; import type { ExecutionRequest } from '../../../types'; import { getSharedWorkflowIds } from '../workflows/workflows.service'; import { encodeNextCursor } from '../../shared/services/pagination.service'; -import { InternalHooks } from '@/InternalHooks'; +import { EventService } from '@/events/event.service'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { ConcurrencyControlService } from '@/concurrency/concurrency-control.service'; @@ -78,9 +78,9 @@ export = { return res.status(404).json({ message: 'Not Found' }); } - void Container.get(InternalHooks).onUserRetrievedExecution({ - user_id: req.user.id, - public_api: true, + Container.get(EventService).emit('user-retrieved-execution', { + userId: req.user.id, + publicApi: true, }); return res.json(replaceCircularReferences(execution)); @@ -95,9 +95,10 @@ export = { status = undefined, includeData = false, workflowId = undefined, + projectId, } = req.query; - const sharedWorkflowsIds = await getSharedWorkflowIds(req.user, ['workflow:read']); + const sharedWorkflowsIds = await getSharedWorkflowIds(req.user, ['workflow:read'], projectId); // user does not have workflows hence no executions // or the execution they are trying to access belongs to a workflow they do not own @@ -129,9 +130,9 @@ export = { const count = await Container.get(ExecutionRepository).getExecutionsCountForPublicApi(filters); - void Container.get(InternalHooks).onUserRetrievedAllExecutions({ - user_id: req.user.id, - public_api: true, + Container.get(EventService).emit('user-retrieved-all-executions', { + userId: req.user.id, + publicApi: true, }); return res.json({ diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/spec/paths/executions.yml b/packages/cli/src/PublicApi/v1/handlers/executions/spec/paths/executions.yml index 8d26492ec3..6fcdaf356e 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/spec/paths/executions.yml +++ b/packages/cli/src/PublicApi/v1/handlers/executions/spec/paths/executions.yml @@ -21,6 +21,14 @@ get: schema: type: string example: '1000' + - name: projectId + in: query + required: false + explode: false + allowReserved: true + schema: + type: string + example: VmwOO9HeTEj20kxM - $ref: '../../../../shared/spec/parameters/limit.yml' - $ref: '../../../../shared/spec/parameters/cursor.yml' responses: diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/projects.handler.ts b/packages/cli/src/PublicApi/v1/handlers/projects/projects.handler.ts new file mode 100644 index 0000000000..e61e808daf --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/projects.handler.ts @@ -0,0 +1,65 @@ +import { globalScope, isLicensed, validCursor } from '../../shared/middlewares/global.middleware'; +import type { Response } from 'express'; +import type { ProjectRequest } from '@/requests'; +import type { PaginatedRequest } from '@/PublicApi/types'; +import Container from 'typedi'; +import { ProjectController } from '@/controllers/project.controller'; +import { ProjectRepository } from '@/databases/repositories/project.repository'; +import { encodeNextCursor } from '../../shared/services/pagination.service'; + +type Create = ProjectRequest.Create; +type Update = ProjectRequest.Update; +type Delete = ProjectRequest.Delete; +type GetAll = PaginatedRequest; + +export = { + createProject: [ + isLicensed('feat:projectRole:admin'), + globalScope('project:create'), + async (req: Create, res: Response) => { + const project = await Container.get(ProjectController).createProject(req); + + return res.status(201).json(project); + }, + ], + updateProject: [ + isLicensed('feat:projectRole:admin'), + globalScope('project:update'), + async (req: Update, res: Response) => { + await Container.get(ProjectController).updateProject(req); + + return res.status(204).send(); + }, + ], + deleteProject: [ + isLicensed('feat:projectRole:admin'), + globalScope('project:delete'), + async (req: Delete, res: Response) => { + await Container.get(ProjectController).deleteProject(req); + + return res.status(204).send(); + }, + ], + getProjects: [ + isLicensed('feat:projectRole:admin'), + globalScope('project:list'), + validCursor, + async (req: GetAll, res: Response) => { + const { offset = 0, limit = 100 } = req.query; + + const [projects, count] = await Container.get(ProjectRepository).findAndCount({ + skip: offset, + take: limit, + }); + + return res.json({ + data: projects, + nextCursor: encodeNextCursor({ + offset, + limit, + numberOfTotalRecords: count, + }), + }); + }, + ], +}; diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.projectId.yml b/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.projectId.yml new file mode 100644 index 0000000000..a5aab19b3d --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.projectId.yml @@ -0,0 +1,43 @@ +delete: + x-eov-operation-id: deleteProject + x-eov-operation-handler: v1/handlers/projects/projects.handler + tags: + - Projects + summary: Delete a project + description: Delete a project from your instance. + parameters: + - $ref: '../schemas/parameters/projectId.yml' + responses: + '204': + description: Operation successful. + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '403': + $ref: '../../../../shared/spec/responses/forbidden.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' +put: + x-eov-operation-id: updateProject + x-eov-operation-handler: v1/handlers/projects/projects.handler + tags: + - Project + summary: Update a project + description: Update a project. + requestBody: + description: Updated project object. + content: + application/json: + schema: + $ref: '../schemas/project.yml' + required: true + responses: + '204': + description: Operation successful. + '400': + $ref: '../../../../shared/spec/responses/badRequest.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '403': + $ref: '../../../../shared/spec/responses/forbidden.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.yml b/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.yml new file mode 100644 index 0000000000..1babd3dd6e --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/spec/paths/projects.yml @@ -0,0 +1,40 @@ +post: + x-eov-operation-id: createProject + x-eov-operation-handler: v1/handlers/projects/projects.handler + tags: + - Projects + summary: Create a project + description: Create a project in your instance. + requestBody: + description: Payload for project to create. + content: + application/json: + schema: + $ref: '../schemas/project.yml' + required: true + responses: + '201': + description: Operation successful. + '400': + $ref: '../../../../shared/spec/responses/badRequest.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' +get: + x-eov-operation-id: getProjects + x-eov-operation-handler: v1/handlers/projects/projects.handler + tags: + - Projects + summary: Retrieve projects + description: Retrieve projects from your instance. + parameters: + - $ref: '../../../../shared/spec/parameters/limit.yml' + - $ref: '../../../../shared/spec/parameters/cursor.yml' + responses: + '200': + description: Operation successful. + content: + application/json: + schema: + $ref: '../schemas/projectList.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/parameters/projectId.yml b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/parameters/projectId.yml new file mode 100644 index 0000000000..32961f4601 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/parameters/projectId.yml @@ -0,0 +1,6 @@ +name: projectId +in: path +description: The ID of the project. +required: true +schema: + type: string diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/project.yml b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/project.yml new file mode 100644 index 0000000000..7a4d2ec432 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/project.yml @@ -0,0 +1,13 @@ +type: object +additionalProperties: false +required: + - name +properties: + id: + type: string + readOnly: true + name: + type: string + type: + type: string + readOnly: true diff --git a/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/projectList.yml b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/projectList.yml new file mode 100644 index 0000000000..7d88be72fb --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/projects/spec/schemas/projectList.yml @@ -0,0 +1,11 @@ +type: object +properties: + data: + type: array + items: + $ref: './project.yml' + nextCursor: + type: string + description: Paginate through projects by setting the cursor parameter to a nextCursor attribute returned by a previous request. Default value fetches the first "page" of the collection. + nullable: true + example: MTIzZTQ1NjctZTg5Yi0xMmQzLWE0NTYtNDI2NjE0MTc0MDA diff --git a/packages/cli/src/PublicApi/v1/handlers/sourceControl/sourceControl.handler.ts b/packages/cli/src/PublicApi/v1/handlers/sourceControl/sourceControl.handler.ts index f54e8bd95d..7a3cf08ec0 100644 --- a/packages/cli/src/PublicApi/v1/handlers/sourceControl/sourceControl.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/sourceControl/sourceControl.handler.ts @@ -10,7 +10,7 @@ import { getTrackingInformationFromPullResult, isSourceControlLicensed, } from '@/environments/sourceControl/sourceControlHelper.ee'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; export = { pull: [ diff --git a/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.role.yml b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.role.yml new file mode 100644 index 0000000000..92993adf7f --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.role.yml @@ -0,0 +1,31 @@ +patch: + x-eov-operation-id: changeRole + x-eov-operation-handler: v1/handlers/users/users.handler.ee + tags: + - User + summary: Change a user's global role + description: Change a user's global role + parameters: + - $ref: '../schemas/parameters/userIdentifier.yml' + requestBody: + description: New role for the user + required: true + content: + application/json: + schema: + type: object + properties: + newRoleName: + type: string + enum: [global:admin, global:member] + required: + - newRoleName + responses: + '200': + description: Operation successful. + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '403': + $ref: '../../../../shared/spec/responses/forbidden.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.yml b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.yml index 0d3c86c4ce..f3dcae0053 100644 --- a/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.yml +++ b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.id.yml @@ -17,3 +17,21 @@ get: $ref: '../schemas/user.yml' '401': $ref: '../../../../shared/spec/responses/unauthorized.yml' +delete: + x-eov-operation-id: deleteUser + x-eov-operation-handler: v1/handlers/users/users.handler.ee + tags: + - User + summary: Delete a user + description: Delete a user from your instance. + parameters: + - $ref: '../schemas/parameters/userIdentifier.yml' + responses: + '204': + description: Operation successful. + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '403': + $ref: '../../../../shared/spec/responses/forbidden.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.yml b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.yml index 1d61843553..e767ab33cc 100644 --- a/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.yml +++ b/packages/cli/src/PublicApi/v1/handlers/users/spec/paths/users.yml @@ -9,6 +9,14 @@ get: - $ref: '../../../../shared/spec/parameters/limit.yml' - $ref: '../../../../shared/spec/parameters/cursor.yml' - $ref: '../schemas/parameters/includeRole.yml' + - name: projectId + in: query + required: false + explode: false + allowReserved: true + schema: + type: string + example: VmwOO9HeTEj20kxM responses: '200': description: Operation successful. @@ -18,3 +26,53 @@ get: $ref: '../schemas/userList.yml' '401': $ref: '../../../../shared/spec/responses/unauthorized.yml' +post: + x-eov-operation-id: createUser + x-eov-operation-handler: v1/handlers/users/users.handler.ee + tags: + - User + summary: Create multiple users + description: Create one or more users. + requestBody: + description: Array of users to be created. + required: true + content: + application/json: + schema: + type: array + items: + type: object + properties: + email: + type: string + format: email + role: + type: string + enum: [global:admin, global:member] + required: + - email + responses: + '200': + description: Operation successful. + content: + application/json: + schema: + type: object + properties: + user: + type: object + properties: + id: + type: string + email: + type: string + inviteAcceptUrl: + type: string + emailSent: + type: boolean + error: + type: string + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '403': + $ref: '../../../../shared/spec/responses/forbidden.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/users/users.handler.ee.ts b/packages/cli/src/PublicApi/v1/handlers/users/users.handler.ee.ts index 96b2d57239..59a7a4c3e3 100644 --- a/packages/cli/src/PublicApi/v1/handlers/users/users.handler.ee.ts +++ b/packages/cli/src/PublicApi/v1/handlers/users/users.handler.ee.ts @@ -6,11 +6,20 @@ import { clean, getAllUsersAndCount, getUser } from './users.service.ee'; import { encodeNextCursor } from '../../shared/services/pagination.service'; import { globalScope, + isLicensed, validCursor, validLicenseWithUserQuota, } from '../../shared/middlewares/global.middleware'; import type { UserRequest } from '@/requests'; -import { InternalHooks } from '@/InternalHooks'; +import { EventService } from '@/events/event.service'; +import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; +import type { Response } from 'express'; +import { InvitationController } from '@/controllers/invitation.controller'; +import { UsersController } from '@/controllers/users.controller'; + +type Create = UserRequest.Invite; +type Delete = UserRequest.Delete; +type ChangeRole = UserRequest.ChangeRole; export = { getUser: [ @@ -28,12 +37,10 @@ export = { }); } - const telemetryData = { - user_id: req.user.id, - public_api: true, - }; - - void Container.get(InternalHooks).onUserRetrievedUser(telemetryData); + Container.get(EventService).emit('user-retrieved-user', { + userId: req.user.id, + publicApi: true, + }); return res.json(clean(user, { includeRole })); }, @@ -43,20 +50,23 @@ export = { validCursor, globalScope(['user:list', 'user:read']), async (req: UserRequest.Get, res: express.Response) => { - const { offset = 0, limit = 100, includeRole = false } = req.query; + const { offset = 0, limit = 100, includeRole = false, projectId } = req.query; + + const _in = projectId + ? await Container.get(ProjectRelationRepository).findUserIdsByProjectId(projectId) + : undefined; const [users, count] = await getAllUsersAndCount({ includeRole, limit, offset, + in: _in, }); - const telemetryData = { - user_id: req.user.id, - public_api: true, - }; - - void Container.get(InternalHooks).onUserRetrievedAllUsers(telemetryData); + Container.get(EventService).emit('user-retrieved-all-users', { + userId: req.user.id, + publicApi: true, + }); return res.json({ data: clean(users, { includeRole }), @@ -68,4 +78,29 @@ export = { }); }, ], + createUser: [ + globalScope('user:create'), + async (req: Create, res: Response) => { + const usersInvited = await Container.get(InvitationController).inviteUser(req); + + return res.status(201).json(usersInvited); + }, + ], + deleteUser: [ + globalScope('user:delete'), + async (req: Delete, res: Response) => { + await Container.get(UsersController).deleteUser(req); + + return res.status(204).send(); + }, + ], + changeRole: [ + isLicensed('feat:advancedPermissions'), + globalScope('user:changeRole'), + async (req: ChangeRole, res: Response) => { + await Container.get(UsersController).changeGlobalRole(req); + + return res.status(204).send(); + }, + ], }; diff --git a/packages/cli/src/PublicApi/v1/handlers/users/users.service.ee.ts b/packages/cli/src/PublicApi/v1/handlers/users/users.service.ee.ts index f7bf661816..e62c946574 100644 --- a/packages/cli/src/PublicApi/v1/handlers/users/users.service.ee.ts +++ b/packages/cli/src/PublicApi/v1/handlers/users/users.service.ee.ts @@ -3,6 +3,8 @@ import { UserRepository } from '@db/repositories/user.repository'; import type { User } from '@db/entities/User'; import pick from 'lodash/pick'; import { validate as uuidValidate } from 'uuid'; +// eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import +import { In } from '@n8n/typeorm'; export async function getUser(data: { withIdentifier: string; @@ -25,9 +27,12 @@ export async function getAllUsersAndCount(data: { includeRole?: boolean; limit?: number; offset?: number; + in?: string[]; }): Promise<[User[], number]> { + const { in: _in } = data; + const users = await Container.get(UserRepository).find({ - where: {}, + where: { ...(_in && { id: In(_in) }) }, skip: data.offset, take: data.limit, }); diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.id.yml b/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.id.yml new file mode 100644 index 0000000000..79c65416b2 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.id.yml @@ -0,0 +1,16 @@ +delete: + x-eov-operation-id: deleteVariable + x-eov-operation-handler: v1/handlers/variables/variables.handler + tags: + - Variables + summary: Delete a variable + description: Delete a variable from your instance. + parameters: + - $ref: '../schemas/parameters/variableId.yml' + responses: + '204': + description: Operation successful. + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.yml b/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.yml new file mode 100644 index 0000000000..7418c2fe05 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/spec/paths/variables.yml @@ -0,0 +1,40 @@ +post: + x-eov-operation-id: createVariable + x-eov-operation-handler: v1/handlers/variables/variables.handler + tags: + - Variables + summary: Create a variable + description: Create a variable in your instance. + requestBody: + description: Payload for variable to create. + content: + application/json: + schema: + $ref: '../schemas/variable.yml' + required: true + responses: + '201': + description: Operation successful. + '400': + $ref: '../../../../shared/spec/responses/badRequest.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' +get: + x-eov-operation-id: getVariables + x-eov-operation-handler: v1/handlers/variables/variables.handler + tags: + - Variables + summary: Retrieve variables + description: Retrieve variables from your instance. + parameters: + - $ref: '../../../../shared/spec/parameters/limit.yml' + - $ref: '../../../../shared/spec/parameters/cursor.yml' + responses: + '200': + description: Operation successful. + content: + application/json: + schema: + $ref: '../schemas/variableList.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/parameters/variableId.yml b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/parameters/variableId.yml new file mode 100644 index 0000000000..a886039d8e --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/parameters/variableId.yml @@ -0,0 +1,6 @@ +name: id +in: path +description: The ID of the variable. +required: true +schema: + type: string diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variable.yml b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variable.yml new file mode 100644 index 0000000000..319ad8d440 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variable.yml @@ -0,0 +1,17 @@ +type: object +additionalProperties: false +required: + - key + - value +properties: + id: + type: string + readOnly: true + key: + type: string + value: + type: string + example: test + type: + type: string + readOnly: true diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variableList.yml b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variableList.yml new file mode 100644 index 0000000000..95e66e180f --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/spec/schemas/variableList.yml @@ -0,0 +1,11 @@ +type: object +properties: + data: + type: array + items: + $ref: './variable.yml' + nextCursor: + type: string + description: Paginate through variables by setting the cursor parameter to a nextCursor attribute returned by a previous request. Default value fetches the first "page" of the collection. + nullable: true + example: MTIzZTQ1NjctZTg5Yi0xMmQzLWE0NTYtNDI2NjE0MTc0MDA diff --git a/packages/cli/src/PublicApi/v1/handlers/variables/variables.handler.ts b/packages/cli/src/PublicApi/v1/handlers/variables/variables.handler.ts new file mode 100644 index 0000000000..3073044eac --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/variables/variables.handler.ts @@ -0,0 +1,55 @@ +import Container from 'typedi'; +import { VariablesRepository } from '@/databases/repositories/variables.repository'; +import { VariablesController } from '@/environments/variables/variables.controller.ee'; +import { globalScope, isLicensed, validCursor } from '../../shared/middlewares/global.middleware'; +import { encodeNextCursor } from '../../shared/services/pagination.service'; +import type { Response } from 'express'; +import type { VariablesRequest } from '@/requests'; +import type { PaginatedRequest } from '@/PublicApi/types'; + +type Create = VariablesRequest.Create; +type Delete = VariablesRequest.Delete; +type GetAll = PaginatedRequest; + +export = { + createVariable: [ + isLicensed('feat:variables'), + globalScope('variable:create'), + async (req: Create, res: Response) => { + await Container.get(VariablesController).createVariable(req); + + res.status(201).send(); + }, + ], + deleteVariable: [ + isLicensed('feat:variables'), + globalScope('variable:delete'), + async (req: Delete, res: Response) => { + await Container.get(VariablesController).deleteVariable(req); + + res.status(204).send(); + }, + ], + getVariables: [ + isLicensed('feat:variables'), + globalScope('variable:list'), + validCursor, + async (req: GetAll, res: Response) => { + const { offset = 0, limit = 100 } = req.query; + + const [variables, count] = await Container.get(VariablesRepository).findAndCount({ + skip: offset, + take: limit, + }); + + return res.json({ + data: variables, + nextCursor: encodeNextCursor({ + offset, + limit, + numberOfTotalRecords: count, + }), + }); + }, + ], +}; diff --git a/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.id.transfer.yml b/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.id.transfer.yml new file mode 100644 index 0000000000..3997647e1d --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.id.transfer.yml @@ -0,0 +1,31 @@ +put: + x-eov-operation-id: transferWorkflow + x-eov-operation-handler: v1/handlers/workflows/workflows.handler + tags: + - Workflow + summary: Transfer a workflow to another project. + description: Transfer a workflow to another project. + parameters: + - $ref: '../schemas/parameters/workflowId.yml' + requestBody: + description: Destination project information for the workflow transfer. + content: + application/json: + schema: + type: object + properties: + destinationProjectId: + type: string + description: The ID of the project to transfer the workflow to. + required: + - destinationProjectId + required: true + responses: + '200': + description: Operation successful. + '400': + $ref: '../../../../shared/spec/responses/badRequest.yml' + '401': + $ref: '../../../../shared/spec/responses/unauthorized.yml' + '404': + $ref: '../../../../shared/spec/responses/notFound.yml' diff --git a/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.yml b/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.yml index 6db149195d..1024e36cb5 100644 --- a/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.yml +++ b/packages/cli/src/PublicApi/v1/handlers/workflows/spec/paths/workflows.yml @@ -52,6 +52,14 @@ get: schema: type: string example: My Workflow + - name: projectId + in: query + required: false + explode: false + allowReserved: true + schema: + type: string + example: VmwOO9HeTEj20kxM - $ref: '../../../../shared/spec/parameters/limit.yml' - $ref: '../../../../shared/spec/parameters/cursor.yml' responses: diff --git a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts index 943d53764d..42c3e269ab 100644 --- a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts @@ -7,11 +7,11 @@ import type { FindOptionsWhere } from '@n8n/typeorm'; import { In, Like, QueryFailedError } from '@n8n/typeorm'; import { v4 as uuid } from 'uuid'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; import config from '@/config'; import { WorkflowEntity } from '@db/entities/WorkflowEntity'; -import { ExternalHooks } from '@/ExternalHooks'; -import { addNodeIds, replaceInvalidCredentials } from '@/WorkflowHelpers'; +import { ExternalHooks } from '@/external-hooks'; +import { addNodeIds, replaceInvalidCredentials } from '@/workflow-helpers'; import type { WorkflowRequest } from '../../../types'; import { projectScope, validCursor } from '../../shared/middlewares/global.middleware'; import { encodeNextCursor } from '../../shared/services/pagination.service'; @@ -26,13 +26,14 @@ import { updateTags, } from './workflows.service'; import { WorkflowService } from '@/workflows/workflow.service'; -import { InternalHooks } from '@/InternalHooks'; -import { WorkflowHistoryService } from '@/workflows/workflowHistory/workflowHistory.service.ee'; +import { WorkflowHistoryService } from '@/workflows/workflow-history/workflow-history.service.ee'; import { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflow.repository'; import { TagRepository } from '@/databases/repositories/tag.repository'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; +import { z } from 'zod'; +import { EnterpriseWorkflowService } from '@/workflows/workflow.service.ee'; export = { createWorkflow: [ @@ -58,15 +59,31 @@ export = { ); await Container.get(ExternalHooks).run('workflow.afterCreate', [createdWorkflow]); - void Container.get(InternalHooks).onWorkflowCreated(req.user, createdWorkflow, project, true); Container.get(EventService).emit('workflow-created', { workflow: createdWorkflow, user: req.user, + publicApi: true, + projectId: project.id, + projectType: project.type, }); return res.json(createdWorkflow); }, ], + transferWorkflow: [ + projectScope('workflow:move', 'workflow'), + async (req: WorkflowRequest.Transfer, res: express.Response) => { + const body = z.object({ destinationProjectId: z.string() }).parse(req.body); + + await Container.get(EnterpriseWorkflowService).transferOne( + req.user, + req.params.workflowId, + body.destinationProjectId, + ); + + res.status(204).send(); + }, + ], deleteWorkflow: [ projectScope('workflow:delete', 'workflow'), async (req: WorkflowRequest.Get, res: express.Response): Promise => { @@ -101,9 +118,9 @@ export = { return res.status(404).json({ message: 'Not Found' }); } - void Container.get(InternalHooks).onUserRetrievedWorkflow({ - user_id: req.user.id, - public_api: true, + Container.get(EventService).emit('user-retrieved-workflow', { + userId: req.user.id, + publicApi: true, }); return res.json(workflow); @@ -112,7 +129,7 @@ export = { getWorkflows: [ validCursor, async (req: WorkflowRequest.GetAll, res: express.Response): Promise => { - const { offset = 0, limit = 100, active, tags, name } = req.query; + const { offset = 0, limit = 100, active, tags, name, projectId } = req.query; const where: FindOptionsWhere = { ...(active !== undefined && { active }), @@ -126,6 +143,19 @@ export = { ); where.id = In(workflowIds); } + + if (projectId) { + const workflows = await Container.get(SharedWorkflowRepository).findAllWorkflowsForUser( + req.user, + ['workflow:read'], + ); + + const workflowIds = workflows + .filter((workflow) => workflow.projectId === projectId) + .map((workflow) => workflow.id); + + where.id = In(workflowIds); + } } else { const options: { workflowIds?: string[] } = {}; @@ -145,6 +175,10 @@ export = { workflows = workflows.filter((wf) => workflowIds.includes(wf.id)); } + if (projectId) { + workflows = workflows.filter((w) => w.projectId === projectId); + } + if (!workflows.length) { return res.status(200).json({ data: [], @@ -163,9 +197,9 @@ export = { ...(!config.getEnv('workflowTagsDisabled') && { relations: ['tags'] }), }); - void Container.get(InternalHooks).onUserRetrievedAllWorkflows({ - user_id: req.user.id, - public_api: true, + Container.get(EventService).emit('user-retrieved-all-workflows', { + userId: req.user.id, + publicApi: true, }); return res.json({ @@ -239,11 +273,10 @@ export = { } await Container.get(ExternalHooks).run('workflow.afterUpdate', [updateData]); - void Container.get(InternalHooks).onWorkflowSaved(req.user, updateData, true); Container.get(EventService).emit('workflow-saved', { user: req.user, - workflowId: updateData.id, - workflowName: updateData.name, + workflow: updateData, + publicApi: true, }); return res.json(updatedWorkflow); diff --git a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.service.ts b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.service.ts index d301e61c93..5fba88ff16 100644 --- a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.service.ts +++ b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.service.ts @@ -8,8 +8,8 @@ import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import type { Project } from '@/databases/entities/Project'; import { TagRepository } from '@db/repositories/tag.repository'; -import { License } from '@/License'; -import { WorkflowSharingService } from '@/workflows/workflowSharing.service'; +import { License } from '@/license'; +import { WorkflowSharingService } from '@/workflows/workflow-sharing.service'; import type { Scope } from '@n8n/permissions'; import config from '@/config'; @@ -17,15 +17,21 @@ function insertIf(condition: boolean, elements: string[]): string[] { return condition ? elements : []; } -export async function getSharedWorkflowIds(user: User, scopes: Scope[]): Promise { +export async function getSharedWorkflowIds( + user: User, + scopes: Scope[], + projectId?: string, +): Promise { if (Container.get(License).isSharingEnabled()) { return await Container.get(WorkflowSharingService).getSharedWorkflowIds(user, { scopes, + projectId, }); } else { return await Container.get(WorkflowSharingService).getSharedWorkflowIds(user, { workflowRoles: ['workflow:owner'], projectRoles: ['project:personalOwner'], + projectId, }); } } diff --git a/packages/cli/src/PublicApi/v1/openapi.yml b/packages/cli/src/PublicApi/v1/openapi.yml index 9d82499835..0a84925734 100644 --- a/packages/cli/src/PublicApi/v1/openapi.yml +++ b/packages/cli/src/PublicApi/v1/openapi.yml @@ -8,7 +8,7 @@ info: email: hello@n8n.io license: name: Sustainable Use License - url: https://github.com/n8n-io/n8n/blob/master/packages/cli/LICENSE.md + url: https://github.com/n8n-io/n8n/blob/master/LICENSE.md version: 1.1.1 externalDocs: description: n8n API documentation @@ -30,6 +30,10 @@ tags: description: Operations about tags - name: SourceControl description: Operations about source control + - name: Variables + description: Operations about variables + - name: Projects + description: Operations about projects paths: /audit: @@ -56,14 +60,28 @@ paths: $ref: './handlers/workflows/spec/paths/workflows.id.activate.yml' /workflows/{id}/deactivate: $ref: './handlers/workflows/spec/paths/workflows.id.deactivate.yml' + /workflows/{id}/transfer: + $ref: './handlers/workflows/spec/paths/workflows.id.transfer.yml' + /credentials/{id}/transfer: + $ref: './handlers/credentials/spec/paths/credentials.id.transfer.yml' /workflows/{id}/tags: $ref: './handlers/workflows/spec/paths/workflows.id.tags.yml' /users: $ref: './handlers/users/spec/paths/users.yml' /users/{id}: $ref: './handlers/users/spec/paths/users.id.yml' + /users/{id}/role: + $ref: './handlers/users/spec/paths/users.id.role.yml' /source-control/pull: $ref: './handlers/sourceControl/spec/paths/sourceControl.yml' + /variables: + $ref: './handlers/variables/spec/paths/variables.yml' + /variables/{id}: + $ref: './handlers/variables/spec/paths/variables.id.yml' + /projects: + $ref: './handlers/projects/spec/paths/projects.yml' + /projects/{projectId}: + $ref: './handlers/projects/spec/paths/projects.projectId.yml' components: schemas: $ref: './shared/spec/schemas/_index.yml' diff --git a/packages/cli/src/PublicApi/v1/shared/middlewares/global.middleware.ts b/packages/cli/src/PublicApi/v1/shared/middlewares/global.middleware.ts index 6fa9bed113..1b70b9770d 100644 --- a/packages/cli/src/PublicApi/v1/shared/middlewares/global.middleware.ts +++ b/packages/cli/src/PublicApi/v1/shared/middlewares/global.middleware.ts @@ -2,13 +2,15 @@ import type express from 'express'; import { Container } from 'typedi'; -import { License } from '@/License'; +import { License } from '@/license'; import type { AuthenticatedRequest } from '@/requests'; import type { PaginatedRequest } from '../../../types'; import { decodeCursor } from '../services/pagination.service'; import type { Scope } from '@n8n/permissions'; -import { userHasScope } from '@/permissions/checkAccess'; +import { userHasScope } from '@/permissions/check-access'; +import type { BooleanLicenseFeature } from '@/Interfaces'; +import { FeatureNotLicensedError } from '@/errors/feature-not-licensed.error'; const UNLIMITED_USERS_QUOTA = -1; @@ -86,3 +88,11 @@ export const validLicenseWithUserQuota = ( return next(); }; + +export const isLicensed = (feature: BooleanLicenseFeature) => { + return async (_: AuthenticatedRequest, res: express.Response, next: express.NextFunction) => { + if (Container.get(License).isFeatureEnabled(feature)) return next(); + + return res.status(403).json({ message: new FeatureNotLicensedError(feature).message }); + }; +}; diff --git a/packages/cli/src/Queue.ts b/packages/cli/src/Queue.ts deleted file mode 100644 index 11cfed839b..0000000000 --- a/packages/cli/src/Queue.ts +++ /dev/null @@ -1,166 +0,0 @@ -import type Bull from 'bull'; -import Container, { Service } from 'typedi'; -import { - ApplicationError, - BINARY_ENCODING, - type IDataObject, - type ExecutionError, - type IExecuteResponsePromiseData, -} from 'n8n-workflow'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import config from '@/config'; -import { OnShutdown } from './decorators/OnShutdown'; -import { HIGHEST_SHUTDOWN_PRIORITY } from './constants'; - -export type JobId = Bull.JobId; -export type Job = Bull.Job; -export type JobQueue = Bull.Queue; - -export interface JobData { - executionId: string; - loadStaticData: boolean; -} - -export interface JobResponse { - success: boolean; - error?: ExecutionError; -} - -export interface WebhookResponse { - executionId: string; - response: IExecuteResponsePromiseData; -} - -@Service() -export class Queue { - private jobQueue: JobQueue; - - /** - * The number of jobs a single server can process concurrently - * Any worker that wants to process executions must first set this to a non-zero value - */ - private concurrency = 0; - - setConcurrency(concurrency: number) { - this.concurrency = concurrency; - // This sets the max event listeners on the jobQueue EventEmitter to prevent the logs getting flooded with MaxListenersExceededWarning - // see: https://github.com/OptimalBits/bull/blob/develop/lib/job.js#L497-L521 - this.jobQueue.setMaxListeners( - 4 + // `close` - 2 + // `error` - 2 + // `global:progress` - concurrency * 2, // 2 global events for every call to `job.finished()` - ); - } - - constructor(private activeExecutions: ActiveExecutions) {} - - async init() { - const { default: Bull } = await import('bull'); - const { RedisClientService } = await import('@/services/redis/redis-client.service'); - - const redisClientService = Container.get(RedisClientService); - - const bullPrefix = config.getEnv('queue.bull.prefix'); - const prefix = redisClientService.toValidPrefix(bullPrefix); - - this.jobQueue = new Bull('jobs', { - prefix, - settings: config.get('queue.bull.settings'), - createClient: (type) => redisClientService.createClient({ type: `${type}(bull)` }), - }); - - this.jobQueue.on('global:progress', (_jobId, progress: WebhookResponse) => { - this.activeExecutions.resolveResponsePromise( - progress.executionId, - this.decodeWebhookResponse(progress.response), - ); - }); - } - - async findRunningJobBy({ executionId }: { executionId: string }) { - const activeOrWaitingJobs = await this.getJobs(['active', 'waiting']); - - return activeOrWaitingJobs.find(({ data }) => data.executionId === executionId) ?? null; - } - - decodeWebhookResponse(response: IExecuteResponsePromiseData): IExecuteResponsePromiseData { - if ( - typeof response === 'object' && - typeof response.body === 'object' && - (response.body as IDataObject)['__@N8nEncodedBuffer@__'] - ) { - response.body = Buffer.from( - (response.body as IDataObject)['__@N8nEncodedBuffer@__'] as string, - BINARY_ENCODING, - ); - } - - return response; - } - - async add(jobData: JobData, jobOptions: object): Promise { - return await this.jobQueue.add(jobData, jobOptions); - } - - async getJob(jobId: JobId): Promise { - return await this.jobQueue.getJob(jobId); - } - - async getJobs(jobTypes: Bull.JobStatus[]): Promise { - return await this.jobQueue.getJobs(jobTypes); - } - - /** - * Get IDs of executions that are currently in progress in the queue. - */ - async getInProgressExecutionIds() { - const inProgressJobs = await this.getJobs(['active', 'waiting']); - - return new Set(inProgressJobs.map((job) => job.data.executionId)); - } - - async process(fn: Bull.ProcessCallbackFunction): Promise { - return await this.jobQueue.process(this.concurrency, fn); - } - - async ping(): Promise { - return await this.jobQueue.client.ping(); - } - - @OnShutdown(HIGHEST_SHUTDOWN_PRIORITY) - // Stop accepting new jobs, `doNotWaitActive` allows reporting progress - async pause(): Promise { - return await this.jobQueue?.pause(true, true); - } - - getBullObjectInstance(): JobQueue { - if (this.jobQueue === undefined) { - // if queue is not initialized yet throw an error, since we do not want to hand around an undefined queue - throw new ApplicationError('Queue is not initialized yet!'); - } - return this.jobQueue; - } - - /** - * - * @param job A Job instance - * @returns boolean true if we were able to securely stop the job - */ - async stopJob(job: Job): Promise { - if (await job.isActive()) { - // Job is already running so tell it to stop - await job.progress(-1); - return true; - } - // Job did not get started yet so remove from queue - try { - await job.remove(); - return true; - } catch (e) { - await job.progress(-1); - } - - return false; - } -} diff --git a/packages/cli/src/TypedEmitter.ts b/packages/cli/src/TypedEmitter.ts new file mode 100644 index 0000000000..176aa9584c --- /dev/null +++ b/packages/cli/src/TypedEmitter.ts @@ -0,0 +1,48 @@ +import { EventEmitter } from 'node:events'; +import debounce from 'lodash/debounce'; + +type Payloads = { + [E in keyof ListenerMap]: unknown; +}; + +type Listener = (payload: Payload) => void; + +export class TypedEmitter> extends EventEmitter { + private debounceWait = 300; // milliseconds + + override on( + eventName: EventName, + listener: Listener, + ) { + return super.on(eventName, listener); + } + + override once( + eventName: EventName, + listener: Listener, + ) { + return super.once(eventName, listener); + } + + override off( + eventName: EventName, + listener: Listener, + ) { + return super.off(eventName, listener); + } + + override emit( + eventName: EventName, + payload?: ListenerMap[EventName], + ): boolean { + return super.emit(eventName, payload); + } + + protected debouncedEmit = debounce( + ( + eventName: EventName, + payload?: ListenerMap[EventName], + ) => super.emit(eventName, payload), + this.debounceWait, + ); +} diff --git a/packages/cli/src/UserManagement/email/index.ts b/packages/cli/src/UserManagement/email/index.ts deleted file mode 100644 index 8c94805eb6..0000000000 --- a/packages/cli/src/UserManagement/email/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { UserManagementMailer } from './UserManagementMailer'; - -export { UserManagementMailer }; diff --git a/packages/cli/test/unit/License.test.ts b/packages/cli/src/__tests__/License.test.ts similarity index 98% rename from packages/cli/test/unit/License.test.ts rename to packages/cli/src/__tests__/License.test.ts index 6d7311656c..712340664f 100644 --- a/packages/cli/test/unit/License.test.ts +++ b/packages/cli/src/__tests__/License.test.ts @@ -2,10 +2,10 @@ import { LicenseManager } from '@n8n_io/license-sdk'; import { InstanceSettings } from 'n8n-core'; import { mock } from 'jest-mock-extended'; import config from '@/config'; -import { License } from '@/License'; -import { Logger } from '@/Logger'; +import { License } from '@/license'; +import { Logger } from '@/logger'; import { N8N_VERSION } from '@/constants'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { OrchestrationService } from '@/services/orchestration.service'; jest.mock('@n8n_io/license-sdk'); diff --git a/packages/cli/test/unit/ActiveExecutions.test.ts b/packages/cli/src/__tests__/active-executions.test.ts similarity index 97% rename from packages/cli/test/unit/ActiveExecutions.test.ts rename to packages/cli/src/__tests__/active-executions.test.ts index b2454de87c..539c160700 100644 --- a/packages/cli/test/unit/ActiveExecutions.test.ts +++ b/packages/cli/src/__tests__/active-executions.test.ts @@ -1,4 +1,4 @@ -import { ActiveExecutions } from '@/ActiveExecutions'; +import { ActiveExecutions } from '@/active-executions'; import PCancelable from 'p-cancelable'; import { v4 as uuid } from 'uuid'; import type { IExecuteResponsePromiseData, IRun } from 'n8n-workflow'; @@ -20,7 +20,10 @@ const executionRepository = mock({ createNewExecution, }); -const concurrencyControl = mockInstance(ConcurrencyControlService, { isEnabled: false }); +const concurrencyControl = mockInstance(ConcurrencyControlService, { + // @ts-expect-error Private property + isEnabled: false, +}); describe('ActiveExecutions', () => { let activeExecutions: ActiveExecutions; diff --git a/packages/cli/test/unit/CredentialTypes.test.ts b/packages/cli/src/__tests__/credential-types.test.ts similarity index 85% rename from packages/cli/test/unit/CredentialTypes.test.ts rename to packages/cli/src/__tests__/credential-types.test.ts index 0ee99c9f8a..b65e5a4aa3 100644 --- a/packages/cli/test/unit/CredentialTypes.test.ts +++ b/packages/cli/src/__tests__/credential-types.test.ts @@ -1,7 +1,7 @@ -import { CredentialTypes } from '@/CredentialTypes'; +import { CredentialTypes } from '@/credential-types'; import { Container } from 'typedi'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; -import { mockInstance } from '../shared/mocking'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { mockInstance } from '@test/mocking'; describe('CredentialTypes', () => { const mockNodesAndCredentials = mockInstance(LoadNodesAndCredentials, { diff --git a/packages/cli/test/unit/CredentialsHelper.test.ts b/packages/cli/src/__tests__/credentials-helper.test.ts similarity index 96% rename from packages/cli/test/unit/CredentialsHelper.test.ts rename to packages/cli/src/__tests__/credentials-helper.test.ts index 19be100b02..61c3f93013 100644 --- a/packages/cli/test/unit/CredentialsHelper.test.ts +++ b/packages/cli/src/__tests__/credentials-helper.test.ts @@ -9,12 +9,12 @@ import type { } from 'n8n-workflow'; import { deepCopy } from 'n8n-workflow'; import { Workflow } from 'n8n-workflow'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import { NodeTypes } from '@/NodeTypes'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; +import { CredentialsHelper } from '@/credentials-helper'; +import { NodeTypes } from '@/node-types'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; describe('CredentialsHelper', () => { mockInstance(CredentialsRepository); diff --git a/packages/cli/test/unit/WaitTracker.test.ts b/packages/cli/src/__tests__/wait-tracker.test.ts similarity index 98% rename from packages/cli/test/unit/WaitTracker.test.ts rename to packages/cli/src/__tests__/wait-tracker.test.ts index 0f8464e18d..ee0697e110 100644 --- a/packages/cli/test/unit/WaitTracker.test.ts +++ b/packages/cli/src/__tests__/wait-tracker.test.ts @@ -1,4 +1,4 @@ -import { WaitTracker } from '@/WaitTracker'; +import { WaitTracker } from '@/wait-tracker'; import { mock } from 'jest-mock-extended'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import type { IExecutionResponse } from '@/Interfaces'; @@ -10,7 +10,7 @@ jest.useFakeTimers(); describe('WaitTracker', () => { const executionRepository = mock(); const multiMainSetup = mock(); - const orchestrationService = new OrchestrationService(mock(), mock(), multiMainSetup); + const orchestrationService = new OrchestrationService(mock(), mock(), mock(), multiMainSetup); const execution = mock({ id: '123', diff --git a/packages/cli/test/unit/WorkflowExecuteAdditionalData.test.ts b/packages/cli/src/__tests__/workflow-execute-additional-data.test.ts similarity index 85% rename from packages/cli/test/unit/WorkflowExecuteAdditionalData.test.ts rename to packages/cli/src/__tests__/workflow-execute-additional-data.test.ts index 2984220637..f51d994493 100644 --- a/packages/cli/test/unit/WorkflowExecuteAdditionalData.test.ts +++ b/packages/cli/src/__tests__/workflow-execute-additional-data.test.ts @@ -1,10 +1,10 @@ import { VariablesService } from '@/environments/variables/variables.service.ee'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { getBase } from '@/WorkflowExecuteAdditionalData'; +import { getBase } from '@/workflow-execute-additional-data'; import Container from 'typedi'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import { SecretsHelper } from '@/SecretsHelpers'; +import { CredentialsHelper } from '@/credentials-helper'; +import { SecretsHelper } from '@/secrets-helpers'; describe('WorkflowExecuteAdditionalData', () => { const messageEventBus = mockInstance(MessageEventBus); diff --git a/packages/cli/test/unit/WorkflowHelpers.test.ts b/packages/cli/src/__tests__/workflow-helpers.test.ts similarity index 95% rename from packages/cli/test/unit/WorkflowHelpers.test.ts rename to packages/cli/src/__tests__/workflow-helpers.test.ts index 1b5da0a4de..32f4933349 100644 --- a/packages/cli/test/unit/WorkflowHelpers.test.ts +++ b/packages/cli/src/__tests__/workflow-helpers.test.ts @@ -1,5 +1,5 @@ import { type Workflow } from 'n8n-workflow'; -import { getExecutionStartNode } from '@/WorkflowHelpers'; +import { getExecutionStartNode } from '@/workflow-helpers'; import type { IWorkflowExecutionDataProcess } from '@/Interfaces'; describe('WorkflowHelpers', () => { diff --git a/packages/cli/test/unit/WorkflowRunner.test.ts b/packages/cli/src/__tests__/workflow-runner.test.ts similarity index 84% rename from packages/cli/test/unit/WorkflowRunner.test.ts rename to packages/cli/src/__tests__/workflow-runner.test.ts index c972d6bb73..56dcf97c7c 100644 --- a/packages/cli/test/unit/WorkflowRunner.test.ts +++ b/packages/cli/src/__tests__/workflow-runner.test.ts @@ -1,14 +1,14 @@ import Container from 'typedi'; import { WorkflowHooks, type ExecutionError, type IWorkflowExecuteHooks } from 'n8n-workflow'; import type { User } from '@db/entities/User'; -import { WorkflowRunner } from '@/WorkflowRunner'; +import { WorkflowRunner } from '@/workflow-runner'; import config from '@/config'; -import * as testDb from '../integration/shared/testDb'; -import { setupTestServer } from '../integration/shared/utils'; -import { createUser } from '../integration/shared/db/users'; -import { createWorkflow } from '../integration/shared/db/workflows'; -import { createExecution } from '../integration/shared/db/executions'; +import * as testDb from '@test-integration/testDb'; +import { setupTestServer } from '@test-integration/utils'; +import { createUser } from '@test-integration/db/users'; +import { createWorkflow } from '@test-integration/db/workflows'; +import { createExecution } from '@test-integration/db/executions'; import { mockInstance } from '@test/mocking'; import { Telemetry } from '@/telemetry'; diff --git a/packages/cli/src/AbstractServer.ts b/packages/cli/src/abstract-server.ts similarity index 70% rename from packages/cli/src/AbstractServer.ts rename to packages/cli/src/abstract-server.ts index b458981c15..f9080080d0 100644 --- a/packages/cli/src/AbstractServer.ts +++ b/packages/cli/src/abstract-server.ts @@ -10,18 +10,19 @@ import config from '@/config'; import { N8N_VERSION, TEMPLATES_DIR, inDevelopment, inTest } from '@/constants'; import * as Db from '@/Db'; import { N8nInstanceType } from '@/Interfaces'; -import { ExternalHooks } from '@/ExternalHooks'; -import { send, sendErrorResponse } from '@/ResponseHelper'; +import { ExternalHooks } from '@/external-hooks'; +import { send, sendErrorResponse } from '@/response-helper'; import { rawBodyReader, bodyParser, corsMiddleware } from '@/middlewares'; -import { TestWebhooks } from '@/TestWebhooks'; -import { WaitingForms } from '@/WaitingForms'; -import { WaitingWebhooks } from '@/WaitingWebhooks'; -import { webhookRequestHandler } from '@/WebhookHelpers'; +import { WaitingForms } from '@/waiting-forms'; +import { TestWebhooks } from '@/webhooks/test-webhooks'; +import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; +import { createWebhookHandlerFor } from '@/webhooks/webhook-request-handler'; +import { LiveWebhooks } from '@/webhooks/live-webhooks'; import { generateHostInstanceId } from './databases/utils/generators'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { ServiceUnavailableError } from './errors/response-errors/service-unavailable.error'; -import { OnShutdown } from '@/decorators/OnShutdown'; -import { ActiveWebhooks } from '@/ActiveWebhooks'; +import { OnShutdown } from '@/decorators/on-shutdown'; +import { GlobalConfig } from '@n8n/config'; @Service() export abstract class AbstractServer { @@ -33,7 +34,7 @@ export abstract class AbstractServer { protected externalHooks: ExternalHooks; - protected protocol = config.getEnv('protocol'); + protected globalConfig = Container.get(GlobalConfig); protected sslKey: string; @@ -73,15 +74,15 @@ export abstract class AbstractServer { this.sslKey = config.getEnv('ssl_key'); this.sslCert = config.getEnv('ssl_cert'); - this.restEndpoint = config.getEnv('endpoints.rest'); + this.restEndpoint = this.globalConfig.endpoints.rest; - this.endpointForm = config.getEnv('endpoints.form'); - this.endpointFormTest = config.getEnv('endpoints.formTest'); - this.endpointFormWaiting = config.getEnv('endpoints.formWaiting'); + this.endpointForm = this.globalConfig.endpoints.form; + this.endpointFormTest = this.globalConfig.endpoints.formTest; + this.endpointFormWaiting = this.globalConfig.endpoints.formWaiting; - this.endpointWebhook = config.getEnv('endpoints.webhook'); - this.endpointWebhookTest = config.getEnv('endpoints.webhookTest'); - this.endpointWebhookWaiting = config.getEnv('endpoints.webhookWaiting'); + this.endpointWebhook = this.globalConfig.endpoints.webhook; + this.endpointWebhookTest = this.globalConfig.endpoints.webhookTest; + this.endpointWebhookWaiting = this.globalConfig.endpoints.webhookWaiting; this.uniqueInstanceId = generateHostInstanceId(instanceType); @@ -133,7 +134,8 @@ export abstract class AbstractServer { } async init(): Promise { - const { app, protocol, sslKey, sslCert } = this; + const { app, sslKey, sslCert } = this; + const { protocol } = this.globalConfig; if (protocol === 'https' && sslKey && sslCert) { const https = await import('https'); @@ -149,25 +151,24 @@ export abstract class AbstractServer { this.server = http.createServer(app); } - const PORT = config.getEnv('port'); - const ADDRESS = config.getEnv('listen_address'); + const { port, listen_address: address } = Container.get(GlobalConfig); this.server.on('error', (error: Error & { code: string }) => { if (error.code === 'EADDRINUSE') { this.logger.info( - `n8n's port ${PORT} is already in use. Do you have another instance of n8n running already?`, + `n8n's port ${port} is already in use. Do you have another instance of n8n running already?`, ); process.exit(1); } }); - await new Promise((resolve) => this.server.listen(PORT, ADDRESS, () => resolve())); + await new Promise((resolve) => this.server.listen(port, address, () => resolve())); this.externalHooks = Container.get(ExternalHooks); await this.setupHealthCheck(); - this.logger.info(`n8n ready on ${ADDRESS}, port ${PORT}`); + this.logger.info(`n8n ready on ${address}, port ${port}`); } async start(): Promise { @@ -180,33 +181,32 @@ export abstract class AbstractServer { // Setup webhook handlers before bodyParser, to let the Webhook node handle binary data in requests if (this.webhooksEnabled) { - const activeWebhooks = Container.get(ActiveWebhooks); + const liveWebhooksRequestHandler = createWebhookHandlerFor(Container.get(LiveWebhooks)); + // Register a handler for live forms + this.app.all(`/${this.endpointForm}/:path(*)`, liveWebhooksRequestHandler); - // Register a handler for active forms - this.app.all(`/${this.endpointForm}/:path(*)`, webhookRequestHandler(activeWebhooks)); - - // Register a handler for active webhooks - this.app.all(`/${this.endpointWebhook}/:path(*)`, webhookRequestHandler(activeWebhooks)); + // Register a handler for live webhooks + this.app.all(`/${this.endpointWebhook}/:path(*)`, liveWebhooksRequestHandler); // Register a handler for waiting forms this.app.all( `/${this.endpointFormWaiting}/:path/:suffix?`, - webhookRequestHandler(Container.get(WaitingForms)), + createWebhookHandlerFor(Container.get(WaitingForms)), ); // Register a handler for waiting webhooks this.app.all( `/${this.endpointWebhookWaiting}/:path/:suffix?`, - webhookRequestHandler(Container.get(WaitingWebhooks)), + createWebhookHandlerFor(Container.get(WaitingWebhooks)), ); } if (this.testWebhooksEnabled) { - const testWebhooks = Container.get(TestWebhooks); + const testWebhooksRequestHandler = createWebhookHandlerFor(Container.get(TestWebhooks)); // Register a handler - this.app.all(`/${this.endpointFormTest}/:path(*)`, webhookRequestHandler(testWebhooks)); - this.app.all(`/${this.endpointWebhookTest}/:path(*)`, webhookRequestHandler(testWebhooks)); + this.app.all(`/${this.endpointFormTest}/:path(*)`, testWebhooksRequestHandler); + this.app.all(`/${this.endpointWebhookTest}/:path(*)`, testWebhooksRequestHandler); } // Block bots from scanning the application @@ -261,14 +261,16 @@ export abstract class AbstractServer { return; } - this.logger.debug(`Shutting down ${this.protocol} server`); + const { protocol } = this.globalConfig; + + this.logger.debug(`Shutting down ${protocol} server`); this.server.close((error) => { if (error) { - this.logger.error(`Error while shutting down ${this.protocol} server`, { error }); + this.logger.error(`Error while shutting down ${protocol} server`, { error }); } - this.logger.debug(`${this.protocol} server shut down`); + this.logger.debug(`${protocol} server shut down`); }); } } diff --git a/packages/cli/src/ActivationErrors.service.ts b/packages/cli/src/activation-errors.service.ts similarity index 100% rename from packages/cli/src/ActivationErrors.service.ts rename to packages/cli/src/activation-errors.service.ts diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/active-executions.ts similarity index 97% rename from packages/cli/src/ActiveExecutions.ts rename to packages/cli/src/active-executions.ts index 97313d5cb2..32f8aac0a7 100644 --- a/packages/cli/src/ActiveExecutions.ts +++ b/packages/cli/src/active-executions.ts @@ -12,6 +12,7 @@ import { ExecutionCancelledError, sleep, } from 'n8n-workflow'; +import { strict as assert } from 'node:assert'; import type { ExecutionPayload, @@ -22,7 +23,7 @@ import type { } from '@/Interfaces'; import { isWorkflowIdValid } from '@/utils'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { ConcurrencyControlService } from './concurrency/concurrency-control.service'; import config from './config'; @@ -74,9 +75,7 @@ export class ActiveExecutions { } executionId = await this.executionRepository.createNewExecution(fullExecutionData); - if (executionId === undefined) { - throw new ApplicationError('There was an issue assigning an execution id to the execution'); - } + assert(executionId); await this.concurrencyControl.throttle({ mode, executionId }); executionStatus = 'running'; diff --git a/packages/cli/src/ActiveWorkflowManager.ts b/packages/cli/src/active-workflow-manager.ts similarity index 97% rename from packages/cli/src/ActiveWorkflowManager.ts rename to packages/cli/src/active-workflow-manager.ts index 1d5050e4d1..0db162a06e 100644 --- a/packages/cli/src/ActiveWorkflowManager.ts +++ b/packages/cli/src/active-workflow-manager.ts @@ -27,28 +27,28 @@ import { } from 'n8n-workflow'; import type { IWorkflowDb } from '@/Interfaces'; -import * as WebhookHelpers from '@/WebhookHelpers'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as WebhookHelpers from '@/webhooks/webhook-helpers'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; -import { ActiveExecutions } from '@/ActiveExecutions'; +import { ActiveExecutions } from '@/active-executions'; import { ExecutionService } from './executions/execution.service'; import { STARTING_NODES, WORKFLOW_REACTIVATE_INITIAL_TIMEOUT, WORKFLOW_REACTIVATE_MAX_TIMEOUT, } from '@/constants'; -import { NodeTypes } from '@/NodeTypes'; -import { ExternalHooks } from '@/ExternalHooks'; -import { WebhookService } from './services/webhook.service'; -import { Logger } from './Logger'; +import { NodeTypes } from '@/node-types'; +import { ExternalHooks } from '@/external-hooks'; +import { WebhookService } from '@/webhooks/webhook.service'; +import { Logger } from './logger'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { OrchestrationService } from '@/services/orchestration.service'; -import { ActivationErrorsService } from '@/ActivationErrors.service'; +import { ActivationErrorsService } from '@/activation-errors.service'; import { ActiveWorkflowsService } from '@/services/activeWorkflows.service'; -import { WorkflowExecutionService } from '@/workflows/workflowExecution.service'; -import { WorkflowStaticDataService } from '@/workflows/workflowStaticData.service'; -import { OnShutdown } from '@/decorators/OnShutdown'; +import { WorkflowExecutionService } from '@/workflows/workflow-execution.service'; +import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; +import { OnShutdown } from '@/decorators/on-shutdown'; interface QueuedActivation { activationMode: WorkflowActivateMode; diff --git a/packages/cli/test/unit/auth/auth.service.test.ts b/packages/cli/src/auth/__tests__/auth.service.test.ts similarity index 87% rename from packages/cli/test/unit/auth/auth.service.test.ts rename to packages/cli/src/auth/__tests__/auth.service.test.ts index 60fdd12126..82c820ac9e 100644 --- a/packages/cli/test/unit/auth/auth.service.test.ts +++ b/packages/cli/src/auth/__tests__/auth.service.test.ts @@ -6,6 +6,7 @@ import { AuthService } from '@/auth/auth.service'; import config from '@/config'; import { AUTH_COOKIE_NAME, Time } from '@/constants'; import type { User } from '@db/entities/User'; +import type { InvalidAuthTokenRepository } from '@db/repositories/invalidAuthToken.repository'; import type { UserRepository } from '@db/repositories/user.repository'; import { JwtService } from '@/services/jwt.service'; import type { UrlService } from '@/services/url.service'; @@ -26,7 +27,15 @@ describe('AuthService', () => { const jwtService = new JwtService(mock()); const urlService = mock(); const userRepository = mock(); - const authService = new AuthService(mock(), mock(), jwtService, urlService, userRepository); + const invalidAuthTokenRepository = mock(); + const authService = new AuthService( + mock(), + mock(), + jwtService, + urlService, + userRepository, + invalidAuthTokenRepository, + ); const now = new Date('2024-02-01T01:23:45.678Z'); jest.useFakeTimers({ now }); @@ -70,16 +79,36 @@ describe('AuthService', () => { it('should 401 if no cookie is set', async () => { req.cookies[AUTH_COOKIE_NAME] = undefined; + await authService.authMiddleware(req, res, next); + + expect(invalidAuthTokenRepository.existsBy).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(401); }); it('should 401 and clear the cookie if the JWT is expired', async () => { req.cookies[AUTH_COOKIE_NAME] = validToken; + invalidAuthTokenRepository.existsBy.mockResolvedValue(false); jest.advanceTimersByTime(365 * Time.days.toMilliseconds); await authService.authMiddleware(req, res, next); + + expect(invalidAuthTokenRepository.existsBy).toHaveBeenCalled(); + expect(userRepository.findOne).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(401); + expect(res.clearCookie).toHaveBeenCalledWith(AUTH_COOKIE_NAME); + }); + + it('should 401 and clear the cookie if the JWT has been invalidated', async () => { + req.cookies[AUTH_COOKIE_NAME] = validToken; + invalidAuthTokenRepository.existsBy.mockResolvedValue(true); + + await authService.authMiddleware(req, res, next); + + expect(invalidAuthTokenRepository.existsBy).toHaveBeenCalled(); + expect(userRepository.findOne).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(401); expect(res.clearCookie).toHaveBeenCalledWith(AUTH_COOKIE_NAME); @@ -88,9 +117,11 @@ describe('AuthService', () => { it('should refresh the cookie before it expires', async () => { req.cookies[AUTH_COOKIE_NAME] = validToken; jest.advanceTimersByTime(6 * Time.days.toMilliseconds); + invalidAuthTokenRepository.existsBy.mockResolvedValue(false); userRepository.findOne.mockResolvedValue(user); await authService.authMiddleware(req, res, next); + expect(next).toHaveBeenCalled(); expect(res.cookie).toHaveBeenCalledWith('n8n-auth', expect.any(String), { httpOnly: true, @@ -302,4 +333,21 @@ describe('AuthService', () => { expect(resolvedUser).toEqual(user); }); }); + + describe('invalidateToken', () => { + const req = mock({ + cookies: { + [AUTH_COOKIE_NAME]: validToken, + }, + }); + + it('should invalidate the token', async () => { + await authService.invalidateToken(req); + + expect(invalidAuthTokenRepository.insert).toHaveBeenCalledWith({ + token: validToken, + expiresAt: new Date('2024-02-08T01:23:45.000Z'), + }); + }); + }); }); diff --git a/packages/cli/src/auth/auth.service.ts b/packages/cli/src/auth/auth.service.ts index ccf562e27e..7e83393f27 100644 --- a/packages/cli/src/auth/auth.service.ts +++ b/packages/cli/src/auth/auth.service.ts @@ -1,4 +1,4 @@ -import { Service } from 'typedi'; +import Container, { Service } from 'typedi'; import type { NextFunction, Response } from 'express'; import { createHash } from 'crypto'; import { JsonWebTokenError, TokenExpiredError } from 'jsonwebtoken'; @@ -6,14 +6,16 @@ import { JsonWebTokenError, TokenExpiredError } from 'jsonwebtoken'; import config from '@/config'; import { AUTH_COOKIE_NAME, RESPONSE_ERROR_MESSAGES, Time } from '@/constants'; import type { User } from '@db/entities/User'; +import { InvalidAuthTokenRepository } from '@db/repositories/invalidAuthToken.repository'; import { UserRepository } from '@db/repositories/user.repository'; import { AuthError } from '@/errors/response-errors/auth.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; -import { License } from '@/License'; -import { Logger } from '@/Logger'; +import { License } from '@/license'; +import { Logger } from '@/logger'; import type { AuthenticatedRequest } from '@/requests'; import { JwtService } from '@/services/jwt.service'; import { UrlService } from '@/services/url.service'; +import { GlobalConfig } from '@n8n/config'; interface AuthJwtPayload { /** User Id */ @@ -33,7 +35,7 @@ interface PasswordResetToken { hash: string; } -const restEndpoint = config.get('endpoints.rest'); +const restEndpoint = Container.get(GlobalConfig).endpoints.rest; // The browser-id check needs to be skipped on these endpoints const skipBrowserIdCheckEndpoints = [ // we need to exclude push endpoint because we can't send custom header on websocket requests @@ -42,10 +44,6 @@ const skipBrowserIdCheckEndpoints = [ // We need to exclude binary-data downloading endpoint because we can't send custom headers on `` tags `/${restEndpoint}/binary-data/`, - - // oAuth callback urls aren't called by the frontend. therefore we can't send custom header on these requests - `/${restEndpoint}/oauth1-credential/callback`, - `/${restEndpoint}/oauth2-credential/callback`, ]; @Service() @@ -56,6 +54,7 @@ export class AuthService { private readonly jwtService: JwtService, private readonly urlService: UrlService, private readonly userRepository: UserRepository, + private readonly invalidAuthTokenRepository: InvalidAuthTokenRepository, ) { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment this.authMiddleware = this.authMiddleware.bind(this); @@ -65,6 +64,8 @@ export class AuthService { const token = req.cookies[AUTH_COOKIE_NAME]; if (token) { try { + const isInvalid = await this.invalidAuthTokenRepository.existsBy({ token }); + if (isInvalid) throw new AuthError('Unauthorized'); req.user = await this.resolveJwt(token, req, res); } catch (error) { if (error instanceof JsonWebTokenError || error instanceof AuthError) { @@ -83,6 +84,22 @@ export class AuthService { res.clearCookie(AUTH_COOKIE_NAME); } + async invalidateToken(req: AuthenticatedRequest) { + const token = req.cookies[AUTH_COOKIE_NAME]; + if (!token) return; + try { + const { exp } = this.jwtService.decode(token); + if (exp) { + await this.invalidAuthTokenRepository.insert({ + token, + expiresAt: new Date(exp * 1000), + }); + } + } catch (e) { + this.logger.warn('failed to invalidate auth token', { error: (e as Error).message }); + } + } + issueCookie(res: Response, user: User, browserId?: string) { // TODO: move this check to the login endpoint in AuthController // If the instance has exceeded its user quota, prevent non-owners from logging in diff --git a/packages/cli/src/auth/methods/email.ts b/packages/cli/src/auth/methods/email.ts index a88d00186b..fb7213ef3b 100644 --- a/packages/cli/src/auth/methods/email.ts +++ b/packages/cli/src/auth/methods/email.ts @@ -1,10 +1,10 @@ import type { User } from '@db/entities/User'; import { PasswordUtility } from '@/services/password.utility'; import { Container } from 'typedi'; -import { isLdapLoginEnabled } from '@/Ldap/helpers.ee'; +import { isLdapLoginEnabled } from '@/ldap/helpers.ee'; import { UserRepository } from '@db/repositories/user.repository'; import { AuthError } from '@/errors/response-errors/auth.error'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; export const handleEmailLogin = async ( email: string, diff --git a/packages/cli/src/auth/methods/ldap.ts b/packages/cli/src/auth/methods/ldap.ts index 09e8f38c87..c3ec2150c7 100644 --- a/packages/cli/src/auth/methods/ldap.ts +++ b/packages/cli/src/auth/methods/ldap.ts @@ -1,7 +1,6 @@ import { Container } from 'typedi'; -import { InternalHooks } from '@/InternalHooks'; -import { LdapService } from '@/Ldap/ldap.service.ee'; +import { LdapService } from '@/ldap/ldap.service.ee'; import { createLdapUserOnLocalDb, getUserByEmail, @@ -10,9 +9,9 @@ import { mapLdapAttributesToUser, createLdapAuthIdentity, updateLdapUserOnLocalDb, -} from '@/Ldap/helpers.ee'; +} from '@/ldap/helpers.ee'; import type { User } from '@db/entities/User'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; export const handleLdapLogin = async ( loginId: string, @@ -51,11 +50,11 @@ export const handleLdapLogin = async ( await updateLdapUserOnLocalDb(identity, ldapAttributesValues); } else { const user = await createLdapUserOnLocalDb(ldapAttributesValues, ldapId); - void Container.get(InternalHooks).onUserSignup(user, { - user_type: 'ldap', - was_disabled_ldap_user: false, + Container.get(EventService).emit('user-signed-up', { + user, + userType: 'ldap', + wasDisabledLdapUser: false, }); - Container.get(EventService).emit('user-signed-up', { user }); return user; } } else { diff --git a/packages/cli/src/commands/audit.ts b/packages/cli/src/commands/audit.ts index 4f6b356028..a447204b67 100644 --- a/packages/cli/src/commands/audit.ts +++ b/packages/cli/src/commands/audit.ts @@ -2,11 +2,11 @@ import { Container } from 'typedi'; import { Flags } from '@oclif/core'; import { ApplicationError } from 'n8n-workflow'; -import { SecurityAuditService } from '@/security-audit/SecurityAudit.service'; +import { SecurityAuditService } from '@/security-audit/security-audit.service'; import { RISK_CATEGORIES } from '@/security-audit/constants'; import config from '@/config'; import type { Risk } from '@/security-audit/types'; -import { BaseCommand } from './BaseCommand'; +import { BaseCommand } from './base-command'; export class SecurityAudit extends BaseCommand { static description = 'Generate a security audit report for this n8n instance'; diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/base-command.ts similarity index 87% rename from packages/cli/src/commands/BaseCommand.ts rename to packages/cli/src/commands/base-command.ts index b00d314b91..db6897c252 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/base-command.ts @@ -4,26 +4,26 @@ import { Command, Errors } from '@oclif/core'; import { GlobalConfig } from '@n8n/config'; import { ApplicationError, ErrorReporterProxy as ErrorReporter, sleep } from 'n8n-workflow'; import { BinaryDataService, InstanceSettings, ObjectStoreService } from 'n8n-core'; -import type { AbstractServer } from '@/AbstractServer'; -import { Logger } from '@/Logger'; +import type { AbstractServer } from '@/abstract-server'; +import { Logger } from '@/logger'; import config from '@/config'; import * as Db from '@/Db'; -import * as CrashJournal from '@/CrashJournal'; +import * as CrashJournal from '@/crash-journal'; import { LICENSE_FEATURES, inDevelopment, inTest } from '@/constants'; -import { initErrorHandling } from '@/ErrorReporting'; -import { ExternalHooks } from '@/ExternalHooks'; -import { NodeTypes } from '@/NodeTypes'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; +import { initErrorHandling } from '@/error-reporting'; +import { ExternalHooks } from '@/external-hooks'; +import { NodeTypes } from '@/node-types'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import type { N8nInstanceType } from '@/Interfaces'; -import { InternalHooks } from '@/InternalHooks'; import { PostHogClient } from '@/posthog'; -import { License } from '@/License'; -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { initExpressionEvaluator } from '@/ExpressionEvaluator'; +import { InternalHooks } from '@/internal-hooks'; +import { License } from '@/license'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; +import { initExpressionEvaluator } from '@/expression-evaluator'; import { generateHostInstanceId } from '@db/utils/generators'; -import { WorkflowHistoryManager } from '@/workflows/workflowHistory/workflowHistoryManager.ee'; -import { ShutdownService } from '@/shutdown/Shutdown.service'; -import { TelemetryEventRelay } from '@/telemetry/telemetry-event-relay.service'; +import { WorkflowHistoryManager } from '@/workflows/workflow-history/workflow-history-manager.ee'; +import { ShutdownService } from '@/shutdown/shutdown.service'; +import { TelemetryEventRelay } from '@/events/telemetry-event-relay'; export abstract class BaseCommand extends Command { protected logger = Container.get(Logger); @@ -44,13 +44,16 @@ export abstract class BaseCommand extends Command { protected license: License; - private globalConfig = Container.get(GlobalConfig); + protected readonly globalConfig = Container.get(GlobalConfig); /** * How long to wait for graceful shutdown before force killing the process. */ protected gracefulShutdownTimeoutInS = config.getEnv('generic.gracefulShutdownTimeout'); + /** Whether to init community packages (if enabled) */ + protected needsCommunityPackages = false; + async init(): Promise { await initErrorHandling(); initExpressionEvaluator(); @@ -111,6 +114,12 @@ export abstract class BaseCommand extends Command { ); } + const { communityPackages } = this.globalConfig.nodes; + if (communityPackages.enabled && this.needsCommunityPackages) { + const { CommunityPackagesService } = await import('@/services/communityPackages.service'); + await Container.get(CommunityPackagesService).checkForMissingPackages(); + } + await Container.get(PostHogClient).init(); await Container.get(InternalHooks).init(); await Container.get(TelemetryEventRelay).init(); @@ -306,7 +315,7 @@ export abstract class BaseCommand extends Command { this.exit(exitCode); } - private onTerminationSignal(signal: string) { + protected onTerminationSignal(signal: string) { return async () => { if (this.shutdownService.isShuttingDown()) { this.logger.info(`Received ${signal}. Already shutting down...`); @@ -324,7 +333,9 @@ export abstract class BaseCommand extends Command { this.logger.info(`Received ${signal}. Shutting down...`); this.shutdownService.shutdown(); - await Promise.all([this.stopProcess(), this.shutdownService.waitForShutdown()]); + await this.shutdownService.waitForShutdown(); + + await this.stopProcess(); clearTimeout(forceShutdownTimer); }; diff --git a/packages/cli/test/unit/commands/db/revert.test.ts b/packages/cli/src/commands/db/__tests__/revert.test.ts similarity index 98% rename from packages/cli/test/unit/commands/db/revert.test.ts rename to packages/cli/src/commands/db/__tests__/revert.test.ts index cea3f89253..1afa8be190 100644 --- a/packages/cli/test/unit/commands/db/revert.test.ts +++ b/packages/cli/src/commands/db/__tests__/revert.test.ts @@ -1,6 +1,6 @@ import { main } from '@/commands/db/revert'; -import { mockInstance } from '../../../shared/mocking'; -import { Logger } from '@/Logger'; +import { mockInstance } from '@test/mocking'; +import { Logger } from '@/logger'; import type { IrreversibleMigration, ReversibleMigration } from '@/databases/types'; import type { Migration, MigrationExecutor } from '@n8n/typeorm'; import { type DataSource } from '@n8n/typeorm'; diff --git a/packages/cli/src/commands/db/revert.ts b/packages/cli/src/commands/db/revert.ts index b0a844086c..c52ce8a35c 100644 --- a/packages/cli/src/commands/db/revert.ts +++ b/packages/cli/src/commands/db/revert.ts @@ -4,7 +4,7 @@ import type { DataSourceOptions as ConnectionOptions } from '@n8n/typeorm'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { MigrationExecutor, DataSource as Connection } from '@n8n/typeorm'; import { Container } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { getConnectionOptions } from '@db/config'; import type { Migration } from '@db/types'; import { wrapMigration } from '@db/utils/migrationHelpers'; diff --git a/packages/cli/src/commands/execute.ts b/packages/cli/src/commands/execute.ts index a375d19c31..0788f4d1db 100644 --- a/packages/cli/src/commands/execute.ts +++ b/packages/cli/src/commands/execute.ts @@ -3,11 +3,11 @@ import { Flags } from '@oclif/core'; import type { IWorkflowBase } from 'n8n-workflow'; import { ApplicationError, ExecutionBaseError } from 'n8n-workflow'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import { WorkflowRunner } from '@/WorkflowRunner'; +import { ActiveExecutions } from '@/active-executions'; +import { WorkflowRunner } from '@/workflow-runner'; import type { IWorkflowExecutionDataProcess } from '@/Interfaces'; import { findCliWorkflowStart, isWorkflowIdValid } from '@/utils'; -import { BaseCommand } from './BaseCommand'; +import { BaseCommand } from './base-command'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { OwnershipService } from '@/services/ownership.service'; @@ -27,6 +27,8 @@ export class Execute extends BaseCommand { }), }; + override needsCommunityPackages = true; + async init() { await super.init(); await this.initBinaryDataService(); diff --git a/packages/cli/src/commands/executeBatch.ts b/packages/cli/src/commands/executeBatch.ts index d8fdbeb8a2..bd2de6b5ed 100644 --- a/packages/cli/src/commands/executeBatch.ts +++ b/packages/cli/src/commands/executeBatch.ts @@ -9,15 +9,15 @@ import { sep } from 'path'; import { diff } from 'json-diff'; import pick from 'lodash/pick'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import { WorkflowRunner } from '@/WorkflowRunner'; +import { ActiveExecutions } from '@/active-executions'; +import { WorkflowRunner } from '@/workflow-runner'; import type { IWorkflowDb, IWorkflowExecutionDataProcess } from '@/Interfaces'; import type { User } from '@db/entities/User'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { OwnershipService } from '@/services/ownership.service'; import { findCliWorkflowStart } from '@/utils'; -import { BaseCommand } from './BaseCommand'; +import { BaseCommand } from './base-command'; import type { IExecutionResult, INodeSpecialCase, @@ -108,6 +108,8 @@ export class ExecuteBatch extends BaseCommand { }), }; + override needsCommunityPackages = true; + /** * Gracefully handles exit. * @param {boolean} skipExit Whether to skip exit or number according to received signal diff --git a/packages/cli/src/commands/export/credentials.ts b/packages/cli/src/commands/export/credentials.ts index 5fb10dcb2c..4af626c735 100644 --- a/packages/cli/src/commands/export/credentials.ts +++ b/packages/cli/src/commands/export/credentials.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import path from 'path'; import { Credentials } from 'n8n-core'; import type { ICredentialsDb, ICredentialsDecryptedDb } from '@/Interfaces'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import Container from 'typedi'; import { ApplicationError } from 'n8n-workflow'; diff --git a/packages/cli/src/commands/export/workflow.ts b/packages/cli/src/commands/export/workflow.ts index 19aa2b9e08..b15484c769 100644 --- a/packages/cli/src/commands/export/workflow.ts +++ b/packages/cli/src/commands/export/workflow.ts @@ -1,7 +1,7 @@ import { Flags } from '@oclif/core'; import fs from 'fs'; import path from 'path'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import Container from 'typedi'; import { ApplicationError } from 'n8n-workflow'; diff --git a/packages/cli/src/commands/import/credentials.ts b/packages/cli/src/commands/import/credentials.ts index 16fd1a5fa5..f36d33ba7b 100644 --- a/packages/cli/src/commands/import/credentials.ts +++ b/packages/cli/src/commands/import/credentials.ts @@ -9,7 +9,7 @@ import type { EntityManager } from '@n8n/typeorm'; import * as Db from '@/Db'; import { SharedCredentials } from '@db/entities/SharedCredentials'; import { CredentialsEntity } from '@db/entities/CredentialsEntity'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import type { ICredentialsEncrypted } from 'n8n-workflow'; import { ApplicationError, jsonParse } from 'n8n-workflow'; import { UM_FIX_INSTRUCTION } from '@/constants'; diff --git a/packages/cli/src/commands/import/workflow.ts b/packages/cli/src/commands/import/workflow.ts index dcf72a9c9a..d7173b591a 100644 --- a/packages/cli/src/commands/import/workflow.ts +++ b/packages/cli/src/commands/import/workflow.ts @@ -11,7 +11,7 @@ import { UserRepository } from '@db/repositories/user.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import type { IWorkflowToImport } from '@/Interfaces'; import { ImportService } from '@/services/import.service'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; diff --git a/packages/cli/src/commands/ldap/reset.ts b/packages/cli/src/commands/ldap/reset.ts index 2f0cc7b305..c91564ead9 100644 --- a/packages/cli/src/commands/ldap/reset.ts +++ b/packages/cli/src/commands/ldap/reset.ts @@ -1,10 +1,10 @@ import Container from 'typedi'; -import { LDAP_DEFAULT_CONFIGURATION, LDAP_FEATURE_NAME } from '@/Ldap/constants'; +import { LDAP_DEFAULT_CONFIGURATION, LDAP_FEATURE_NAME } from '@/ldap/constants'; import { AuthIdentityRepository } from '@db/repositories/authIdentity.repository'; import { AuthProviderSyncHistoryRepository } from '@db/repositories/authProviderSyncHistory.repository'; import { SettingsRepository } from '@db/repositories/settings.repository'; import { UserRepository } from '@db/repositories/user.repository'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { Flags } from '@oclif/core'; import { ApplicationError } from 'n8n-workflow'; import { ProjectRepository } from '@/databases/repositories/project.repository'; diff --git a/packages/cli/src/commands/license/clear.ts b/packages/cli/src/commands/license/clear.ts index 808784bdfd..8439417471 100644 --- a/packages/cli/src/commands/license/clear.ts +++ b/packages/cli/src/commands/license/clear.ts @@ -1,8 +1,8 @@ import { Container } from 'typedi'; import { SETTINGS_LICENSE_CERT_KEY } from '@/constants'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { SettingsRepository } from '@db/repositories/settings.repository'; -import { License } from '@/License'; +import { License } from '@/license'; export class ClearLicenseCommand extends BaseCommand { static description = 'Clear license'; diff --git a/packages/cli/src/commands/license/info.ts b/packages/cli/src/commands/license/info.ts index a4e9e42cf2..6789b4ff41 100644 --- a/packages/cli/src/commands/license/info.ts +++ b/packages/cli/src/commands/license/info.ts @@ -1,6 +1,6 @@ import { Container } from 'typedi'; -import { License } from '@/License'; -import { BaseCommand } from '../BaseCommand'; +import { License } from '@/license'; +import { BaseCommand } from '../base-command'; export class LicenseInfoCommand extends BaseCommand { static description = 'Print license information'; diff --git a/packages/cli/src/commands/list/workflow.ts b/packages/cli/src/commands/list/workflow.ts index 2d33de19e3..35fd073f35 100644 --- a/packages/cli/src/commands/list/workflow.ts +++ b/packages/cli/src/commands/list/workflow.ts @@ -1,7 +1,7 @@ import Container from 'typedi'; import { Flags } from '@oclif/core'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; export class ListWorkflowCommand extends BaseCommand { static description = '\nList workflows'; diff --git a/packages/cli/src/commands/mfa/disable.ts b/packages/cli/src/commands/mfa/disable.ts index dceb81b8b9..acc3439e3d 100644 --- a/packages/cli/src/commands/mfa/disable.ts +++ b/packages/cli/src/commands/mfa/disable.ts @@ -1,7 +1,7 @@ import Container from 'typedi'; import { Flags } from '@oclif/core'; import { AuthUserRepository } from '@db/repositories/authUser.repository'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; export class DisableMFACommand extends BaseCommand { static description = 'Disable MFA authentication for a user'; diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index b1ebd50393..96e3849860 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -8,17 +8,15 @@ import { createReadStream, createWriteStream, existsSync } from 'fs'; import { pipeline } from 'stream/promises'; import replaceStream from 'replacestream'; import glob from 'fast-glob'; -import { GlobalConfig } from '@n8n/config'; import { jsonParse, randomString } from 'n8n-workflow'; import config from '@/config'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; -import { Server } from '@/Server'; +import { ActiveExecutions } from '@/active-executions'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; +import { Server } from '@/server'; import { EDITOR_UI_DIST_DIR, LICENSE_FEATURES } from '@/constants'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { InternalHooks } from '@/InternalHooks'; -import { License } from '@/License'; +import { License } from '@/license'; import { OrchestrationService } from '@/services/orchestration.service'; import { OrchestrationHandlerMainService } from '@/services/orchestration/main/orchestration.handler.main.service'; import { PruningService } from '@/services/pruning.service'; @@ -26,14 +24,13 @@ import { UrlService } from '@/services/url.service'; import { SettingsRepository } from '@db/repositories/settings.repository'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { FeatureNotLicensedError } from '@/errors/feature-not-licensed.error'; -import { WaitTracker } from '@/WaitTracker'; -import { BaseCommand } from './BaseCommand'; +import { WaitTracker } from '@/wait-tracker'; +import { BaseCommand } from './base-command'; import type { IWorkflowExecutionDataProcess } from '@/Interfaces'; import { ExecutionService } from '@/executions/execution.service'; import { OwnershipService } from '@/services/ownership.service'; -import { WorkflowRunner } from '@/WorkflowRunner'; -import { ExecutionRecoveryService } from '@/executions/execution-recovery.service'; -import { EventService } from '@/eventbus/event.service'; +import { WorkflowRunner } from '@/workflow-runner'; +import { EventService } from '@/events/event.service'; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires const open = require('open'); @@ -68,6 +65,8 @@ export class Start extends BaseCommand { protected server = Container.get(Server); + override needsCommunityPackages = true; + constructor(argv: string[], cmdConfig: Config) { super(argv, cmdConfig); this.setInstanceType('main'); @@ -109,7 +108,7 @@ export class Start extends BaseCommand { await Container.get(OrchestrationService).shutdown(); } - await Container.get(InternalHooks).onN8nStop(); + Container.get(EventService).emit('instance-stopped'); await Container.get(ActiveExecutions).shutdown(); @@ -124,8 +123,7 @@ export class Start extends BaseCommand { private async generateStaticAssets() { // Read the index file and replace the path placeholder - const n8nPath = config.getEnv('path'); - const restEndpoint = config.getEnv('endpoints.rest'); + const n8nPath = this.globalConfig.path; const hooksUrls = config.getEnv('externalFrontendHooksUrls'); let scriptsString = ''; @@ -151,7 +149,9 @@ export class Start extends BaseCommand { ]; if (filePath.endsWith('index.html')) { streams.push( - replaceStream('{{REST_ENDPOINT}}', restEndpoint, { ignoreCase: false }), + replaceStream('{{REST_ENDPOINT}}', this.globalConfig.endpoints.rest, { + ignoreCase: false, + }), replaceStream(closingTitleTag, closingTitleTag + scriptsString, { ignoreCase: false, }), @@ -176,6 +176,22 @@ export class Start extends BaseCommand { this.logger.debug(`Queue mode id: ${this.queueModeId}`); } + const { flags } = await this.parse(Start); + const { communityPackages } = this.globalConfig.nodes; + // cli flag overrides the config env variable + if (flags.reinstallMissingPackages) { + if (communityPackages.enabled) { + this.logger.warn( + '`--reinstallMissingPackages` is deprecated: Please use the env variable `N8N_REINSTALL_MISSING_PACKAGES` instead', + ); + communityPackages.reinstallMissing = true; + } else { + this.logger.warn( + '`--reinstallMissingPackages` was passed, but community packages are disabled', + ); + } + } + await super.init(); this.activeWorkflowManager = Container.get(ActiveWorkflowManager); @@ -184,10 +200,7 @@ export class Start extends BaseCommand { await this.initOrchestration(); this.logger.debug('Orchestration init complete'); - if ( - !config.getEnv('license.autoRenewEnabled') && - config.getEnv('multiMainSetup.instanceType') === 'leader' - ) { + if (!config.getEnv('license.autoRenewEnabled') && this.instanceSettings.isLeader) { this.logger.warn( 'Automatic license renewal is disabled. The license will not renew automatically, and access to licensed features may be lost!', ); @@ -204,14 +217,14 @@ export class Start extends BaseCommand { this.initWorkflowHistory(); this.logger.debug('Workflow history init complete'); - if (!config.getEnv('endpoints.disableUi')) { + if (!this.globalConfig.endpoints.disableUi) { await this.generateStaticAssets(); } } async initOrchestration() { if (config.getEnv('executions.mode') === 'regular') { - config.set('multiMainSetup.instanceType', 'leader'); + this.instanceSettings.markAsLeader(); return; } @@ -226,7 +239,10 @@ export class Start extends BaseCommand { await orchestrationService.init(); - await Container.get(OrchestrationHandlerMainService).init(); + await Container.get(OrchestrationHandlerMainService).initWithOptions({ + queueModeId: this.queueModeId, + redisPublisher: Container.get(OrchestrationService).redisPublisher, + }); if (!orchestrationService.isMultiMainSetupEnabled) return; @@ -252,18 +268,9 @@ export class Start extends BaseCommand { config.set(setting.key, jsonParse(setting.value, { fallbackValue: setting.value })); }); - const globalConfig = Container.get(GlobalConfig); - - if (globalConfig.nodes.communityPackages.enabled) { - const { CommunityPackagesService } = await import('@/services/communityPackages.service'); - await Container.get(CommunityPackagesService).setMissingPackages({ - reinstallMissingPackages: flags.reinstallMissingPackages, - }); - } - - const { type: dbType } = globalConfig.database; + const { type: dbType } = this.globalConfig.database; if (dbType === 'sqlite') { - const shouldRunVacuum = globalConfig.database.sqlite.executeVacuumOnStartup; + const shouldRunVacuum = this.globalConfig.database.sqlite.executeVacuumOnStartup; if (shouldRunVacuum) { await Container.get(ExecutionRepository).query('VACUUM;'); } @@ -283,7 +290,7 @@ export class Start extends BaseCommand { } const { default: localtunnel } = await import('@n8n/localtunnel'); - const port = config.getEnv('port'); + const { port } = this.globalConfig; const webhookTunnel = await localtunnel(port, { host: 'https://hooks.n8n.cloud', @@ -300,7 +307,6 @@ export class Start extends BaseCommand { await this.server.start(); Container.get(PruningService).init(); - Container.get(ExecutionRecoveryService).init(); if (config.getEnv('executions.mode') === 'regular') { await this.runEnqueuedExecutions(); @@ -327,7 +333,7 @@ export class Start extends BaseCommand { this.openBrowser(); } else if (key.charCodeAt(0) === 3) { // Ctrl + c got pressed - void this.stopProcess(); + void this.onTerminationSignal('SIGINT')(); } else { // When anything else got pressed, record it and send it on enter into the child process diff --git a/packages/cli/src/commands/update/workflow.ts b/packages/cli/src/commands/update/workflow.ts index f365db2d98..35a3ac016c 100644 --- a/packages/cli/src/commands/update/workflow.ts +++ b/packages/cli/src/commands/update/workflow.ts @@ -1,7 +1,7 @@ import { Container } from 'typedi'; import { Flags } from '@oclif/core'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; export class UpdateWorkflowCommand extends BaseCommand { static description = 'Update workflows'; diff --git a/packages/cli/src/commands/user-management/reset.ts b/packages/cli/src/commands/user-management/reset.ts index 30f60af0a8..992d367e13 100644 --- a/packages/cli/src/commands/user-management/reset.ts +++ b/packages/cli/src/commands/user-management/reset.ts @@ -6,7 +6,7 @@ import { SettingsRepository } from '@db/repositories/settings.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { UserRepository } from '@db/repositories/user.repository'; -import { BaseCommand } from '../BaseCommand'; +import { BaseCommand } from '../base-command'; import { ProjectRepository } from '@/databases/repositories/project.repository'; const defaultUserProps = { diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 5b72c1eb86..36e898761c 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -3,10 +3,9 @@ import { Flags, type Config } from '@oclif/core'; import { ApplicationError } from 'n8n-workflow'; import config from '@/config'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import { WebhookServer } from '@/WebhookServer'; -import { Queue } from '@/Queue'; -import { BaseCommand } from './BaseCommand'; +import { ActiveExecutions } from '@/active-executions'; +import { WebhookServer } from '@/webhooks/webhook-server'; +import { BaseCommand } from './base-command'; import { OrchestrationWebhookService } from '@/services/orchestration/webhook/orchestration.webhook.service'; import { OrchestrationHandlerWebhookService } from '@/services/orchestration/webhook/orchestration.handler.webhook.service'; @@ -22,6 +21,8 @@ export class Webhook extends BaseCommand { protected server = Container.get(WebhookServer); + override needsCommunityPackages = true; + constructor(argv: string[], cmdConfig: Config) { super(argv, cmdConfig); this.setInstanceType('webhook'); @@ -84,7 +85,7 @@ export class Webhook extends BaseCommand { await this.initExternalHooks(); this.logger.debug('External hooks init complete'); await this.initExternalSecrets(); - this.logger.debug('External seecrets init complete'); + this.logger.debug('External secrets init complete'); } async run() { @@ -94,7 +95,8 @@ export class Webhook extends BaseCommand { ); } - await Container.get(Queue).init(); + const { ScalingService } = await import('@/scaling/scaling.service'); + await Container.get(ScalingService).setupQueue(); await this.server.start(); this.logger.debug(`Webhook listener ID: ${this.server.uniqueInstanceId}`); this.logger.info('Webhook listener waiting for requests.'); diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 4f582904cd..594b2f5668 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -2,35 +2,25 @@ import { Container } from 'typedi'; import { Flags, type Config } from '@oclif/core'; import express from 'express'; import http from 'http'; -import type PCancelable from 'p-cancelable'; -import { GlobalConfig } from '@n8n/config'; -import { WorkflowExecute } from 'n8n-core'; -import type { ExecutionStatus, IExecuteResponsePromiseData, INodeTypes, IRun } from 'n8n-workflow'; -import { Workflow, sleep, ApplicationError } from 'n8n-workflow'; +import { ApplicationError } from 'n8n-workflow'; import * as Db from '@/Db'; -import * as ResponseHelper from '@/ResponseHelper'; -import * as WebhookHelpers from '@/WebhookHelpers'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as ResponseHelper from '@/response-helper'; import config from '@/config'; -import type { Job, JobId, JobResponse, WebhookResponse } from '@/Queue'; -import { Queue } from '@/Queue'; +import type { ScalingService } from '@/scaling/scaling.service'; import { N8N_VERSION, inTest } from '@/constants'; -import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { WorkflowRepository } from '@db/repositories/workflow.repository'; import type { ICredentialsOverwrite } from '@/Interfaces'; -import { CredentialsOverwrites } from '@/CredentialsOverwrites'; +import { CredentialsOverwrites } from '@/credentials-overwrites'; import { rawBodyReader, bodyParser } from '@/middlewares'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; import type { RedisServicePubSubSubscriber } from '@/services/redis/RedisServicePubSubSubscriber'; import { EventMessageGeneric } from '@/eventbus/EventMessageClasses/EventMessageGeneric'; import { OrchestrationHandlerWorkerService } from '@/services/orchestration/worker/orchestration.handler.worker.service'; import { OrchestrationWorkerService } from '@/services/orchestration/worker/orchestration.worker.service'; -import type { WorkerJobStatusSummary } from '@/services/orchestration/worker/types'; import { ServiceUnavailableError } from '@/errors/response-errors/service-unavailable.error'; -import { BaseCommand } from './BaseCommand'; -import { MaxStalledCountError } from '@/errors/max-stalled-count.error'; -import { AuditEventRelay } from '@/eventbus/audit-event-relay.service'; +import { BaseCommand } from './base-command'; +import { JobProcessor } from '@/scaling/job-processor'; +import { LogStreamingEventRelay } from '@/events/log-streaming-event-relay'; export class Worker extends BaseCommand { static description = '\nStarts a n8n worker'; @@ -45,18 +35,22 @@ export class Worker extends BaseCommand { }), }; - static runningJobs: { - [key: string]: PCancelable; - } = {}; + /** + * How many jobs this worker may run concurrently. + * + * Taken from env var `N8N_CONCURRENCY_PRODUCTION_LIMIT` if set to a value + * other than -1, else taken from `--concurrency` flag. + */ + concurrency: number; - static runningJobsSummary: { - [jobId: string]: WorkerJobStatusSummary; - } = {}; + scalingService: ScalingService; - static jobQueue: Queue; + jobProcessor: JobProcessor; redisSubscriber: RedisServicePubSubSubscriber; + override needsCommunityPackages = true; + /** * Stop n8n in a graceful way. * Make for example sure that all the webhooks from third party services @@ -67,23 +61,6 @@ export class Worker extends BaseCommand { try { await this.externalHooks?.run('n8n.stop', []); - - const hardStopTimeMs = Date.now() + this.gracefulShutdownTimeoutInS * 1000; - - // Wait for active workflow executions to finish - let count = 0; - while (Object.keys(Worker.runningJobs).length !== 0) { - if (count++ % 4 === 0) { - const waitLeft = Math.ceil((hardStopTimeMs - Date.now()) / 1000); - this.logger.info( - `Waiting for ${ - Object.keys(Worker.runningJobs).length - } active executions to finish... (max wait ${waitLeft} more seconds)`, - ); - } - - await sleep(500); - } } catch (error) { await this.exitWithCrash('There was an error shutting down n8n.', error); } @@ -91,143 +68,6 @@ export class Worker extends BaseCommand { await this.exitSuccessFully(); } - async runJob(job: Job, nodeTypes: INodeTypes): Promise { - const { executionId, loadStaticData } = job.data; - const executionRepository = Container.get(ExecutionRepository); - const fullExecutionData = await executionRepository.findSingleExecution(executionId, { - includeData: true, - unflattenData: true, - }); - - if (!fullExecutionData) { - this.logger.error( - `Worker failed to find data of execution "${executionId}" in database. Cannot continue.`, - { executionId }, - ); - throw new ApplicationError( - 'Unable to find data of execution in database. Aborting execution.', - { extra: { executionId } }, - ); - } - const workflowId = fullExecutionData.workflowData.id; - - this.logger.info( - `Start job: ${job.id} (Workflow ID: ${workflowId} | Execution: ${executionId})`, - ); - await executionRepository.updateStatus(executionId, 'running'); - - let { staticData } = fullExecutionData.workflowData; - if (loadStaticData) { - const workflowData = await Container.get(WorkflowRepository).findOne({ - select: ['id', 'staticData'], - where: { - id: workflowId, - }, - }); - if (workflowData === null) { - this.logger.error( - 'Worker execution failed because workflow could not be found in database.', - { workflowId, executionId }, - ); - throw new ApplicationError('Workflow could not be found', { extra: { workflowId } }); - } - staticData = workflowData.staticData; - } - - const workflowSettings = fullExecutionData.workflowData.settings ?? {}; - - let workflowTimeout = workflowSettings.executionTimeout ?? config.getEnv('executions.timeout'); // initialize with default - - let executionTimeoutTimestamp: number | undefined; - if (workflowTimeout > 0) { - workflowTimeout = Math.min(workflowTimeout, config.getEnv('executions.maxTimeout')); - executionTimeoutTimestamp = Date.now() + workflowTimeout * 1000; - } - - const workflow = new Workflow({ - id: workflowId, - name: fullExecutionData.workflowData.name, - nodes: fullExecutionData.workflowData.nodes, - connections: fullExecutionData.workflowData.connections, - active: fullExecutionData.workflowData.active, - nodeTypes, - staticData, - settings: fullExecutionData.workflowData.settings, - }); - - const additionalData = await WorkflowExecuteAdditionalData.getBase( - undefined, - undefined, - executionTimeoutTimestamp, - ); - additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( - fullExecutionData.mode, - job.data.executionId, - fullExecutionData.workflowData, - { - retryOf: fullExecutionData.retryOf as string, - }, - ); - - additionalData.hooks.hookFunctions.sendResponse = [ - async (response: IExecuteResponsePromiseData): Promise => { - const progress: WebhookResponse = { - executionId, - response: WebhookHelpers.encodeWebhookResponse(response), - }; - await job.progress(progress); - }, - ]; - - additionalData.executionId = executionId; - - additionalData.setExecutionStatus = (status: ExecutionStatus) => { - // Can't set the status directly in the queued worker, but it will happen in InternalHook.onWorkflowPostExecute - this.logger.debug(`Queued worker execution status for ${executionId} is "${status}"`); - }; - - let workflowExecute: WorkflowExecute; - let workflowRun: PCancelable; - if (fullExecutionData.data !== undefined) { - workflowExecute = new WorkflowExecute( - additionalData, - fullExecutionData.mode, - fullExecutionData.data, - ); - workflowRun = workflowExecute.processRunExecutionData(workflow); - } else { - // Execute all nodes - // Can execute without webhook so go on - workflowExecute = new WorkflowExecute(additionalData, fullExecutionData.mode); - workflowRun = workflowExecute.run(workflow); - } - - Worker.runningJobs[job.id] = workflowRun; - Worker.runningJobsSummary[job.id] = { - jobId: job.id.toString(), - executionId, - workflowId: fullExecutionData.workflowId ?? '', - workflowName: fullExecutionData.workflowData.name, - mode: fullExecutionData.mode, - startedAt: fullExecutionData.startedAt, - retryOf: fullExecutionData.retryOf ?? '', - status: fullExecutionData.status, - }; - - // Wait till the execution is finished - await workflowRun; - - delete Worker.runningJobs[job.id]; - delete Worker.runningJobsSummary[job.id]; - - // do NOT call workflowExecuteAfter hook here, since it is being called from processSuccessExecution() - // already! - - return { - success: true, - }; - } - constructor(argv: string[], cmdConfig: Config) { super(argv, cmdConfig); @@ -245,7 +85,7 @@ export class Worker extends BaseCommand { const { QUEUE_WORKER_TIMEOUT } = process.env; if (QUEUE_WORKER_TIMEOUT) { this.gracefulShutdownTimeoutInS = - parseInt(QUEUE_WORKER_TIMEOUT, 10) || config.default('queue.bull.gracefulShutdownTimeout'); + parseInt(QUEUE_WORKER_TIMEOUT, 10) || this.globalConfig.queue.bull.gracefulShutdownTimeout; this.logger.warn( 'QUEUE_WORKER_TIMEOUT has been deprecated. Rename it to N8N_GRACEFUL_SHUTDOWN_TIMEOUT.', ); @@ -255,6 +95,7 @@ export class Worker extends BaseCommand { this.logger.debug('Starting n8n worker...'); this.logger.debug(`Queue mode id: ${this.queueModeId}`); + await this.setConcurrency(); await super.init(); await this.initLicense(); @@ -267,8 +108,7 @@ export class Worker extends BaseCommand { this.logger.debug('External secrets init complete'); await this.initEventBus(); this.logger.debug('Event bus init complete'); - await this.initQueue(); - this.logger.debug('Queue init complete'); + await this.initScalingService(); await this.initOrchestration(); this.logger.debug('Orchestration init complete'); @@ -286,7 +126,7 @@ export class Worker extends BaseCommand { await Container.get(MessageEventBus).initialize({ workerId: this.queueModeId, }); - Container.get(AuditEventRelay).init(); + Container.get(LogStreamingEventRelay).init(); } /** @@ -300,84 +140,32 @@ export class Worker extends BaseCommand { await Container.get(OrchestrationHandlerWorkerService).initWithOptions({ queueModeId: this.queueModeId, redisPublisher: Container.get(OrchestrationWorkerService).redisPublisher, - getRunningJobIds: () => Object.keys(Worker.runningJobs), - getRunningJobsSummary: () => Object.values(Worker.runningJobsSummary), + getRunningJobIds: () => this.jobProcessor.getRunningJobIds(), + getRunningJobsSummary: () => this.jobProcessor.getRunningJobsSummary(), }); } - async initQueue() { + async setConcurrency() { const { flags } = await this.parse(Worker); - const redisConnectionTimeoutLimit = config.getEnv('queue.bull.redis.timeoutThreshold'); - - this.logger.debug( - `Opening Redis connection to listen to messages with timeout ${redisConnectionTimeoutLimit}`, - ); - - Worker.jobQueue = Container.get(Queue); - await Worker.jobQueue.init(); - this.logger.debug('Queue singleton ready'); - const envConcurrency = config.getEnv('executions.concurrency.productionLimit'); - const concurrency = envConcurrency !== -1 ? envConcurrency : flags.concurrency; - Worker.jobQueue.setConcurrency(concurrency); - void Worker.jobQueue.process(async (job) => await this.runJob(job, this.nodeTypes)); + this.concurrency = envConcurrency !== -1 ? envConcurrency : flags.concurrency; + } - Worker.jobQueue.getBullObjectInstance().on('global:progress', (jobId: JobId, progress) => { - // Progress of a job got updated which does get used - // to communicate that a job got canceled. + async initScalingService() { + const { ScalingService } = await import('@/scaling/scaling.service'); + this.scalingService = Container.get(ScalingService); - if (progress === -1) { - // Job has to get canceled - if (Worker.runningJobs[jobId] !== undefined) { - // Job is processed by current worker so cancel - Worker.runningJobs[jobId].cancel(); - delete Worker.runningJobs[jobId]; - } - } - }); + await this.scalingService.setupQueue(); - let lastTimer = 0; - let cumulativeTimeout = 0; - Worker.jobQueue.getBullObjectInstance().on('error', (error: Error) => { - if (error.toString().includes('ECONNREFUSED')) { - const now = Date.now(); - if (now - lastTimer > 30000) { - // Means we had no timeout at all or last timeout was temporary and we recovered - lastTimer = now; - cumulativeTimeout = 0; - } else { - cumulativeTimeout += now - lastTimer; - lastTimer = now; - if (cumulativeTimeout > redisConnectionTimeoutLimit) { - this.logger.error( - `Unable to connect to Redis after ${redisConnectionTimeoutLimit}. Exiting process.`, - ); - process.exit(1); - } - } - this.logger.warn('Redis unavailable - trying to reconnect...'); - } else if (error.toString().includes('Error initializing Lua scripts')) { - // This is a non-recoverable error - // Happens when worker starts and Redis is unavailable - // Even if Redis comes back online, worker will be zombie - this.logger.error('Error initializing worker.'); - process.exit(2); - } else { - this.logger.error('Error from queue: ', error); + this.scalingService.setupWorker(this.concurrency); - if (error.message.includes('job stalled more than maxStalledCount')) { - throw new MaxStalledCountError(error); - } - - throw error; - } - }); + this.jobProcessor = Container.get(JobProcessor); } async setupHealthMonitor() { - const port = config.getEnv('queue.health.port'); + const { port } = this.globalConfig.queue.health; const app = express(); app.disable('x-powered-by'); @@ -409,7 +197,7 @@ export class Worker extends BaseCommand { // if it loses the connection to redis try { // Redis ping - await Worker.jobQueue.ping(); + await this.scalingService.pingQueue(); } catch (e) { this.logger.error('No Redis connection!', e as Error); const error = new ServiceUnavailableError('No Redis connection!'); @@ -429,8 +217,7 @@ export class Worker extends BaseCommand { let presetCredentialsLoaded = false; - const globalConfig = Container.get(GlobalConfig); - const endpointPresetCredentials = globalConfig.credentials.overwrite.endpoint; + const endpointPresetCredentials = this.globalConfig.credentials.overwrite.endpoint; if (endpointPresetCredentials !== '') { // POST endpoint to set preset credentials app.post( @@ -476,18 +263,16 @@ export class Worker extends BaseCommand { } async run() { - const { flags } = await this.parse(Worker); - this.logger.info('\nn8n worker is now ready'); this.logger.info(` * Version: ${N8N_VERSION}`); - this.logger.info(` * Concurrency: ${flags.concurrency}`); + this.logger.info(` * Concurrency: ${this.concurrency}`); this.logger.info(''); - if (config.getEnv('queue.health.active')) { + if (this.globalConfig.queue.health.active) { await this.setupHealthMonitor(); } - if (process.stdout.isTTY) { + if (!inTest && process.stdout.isTTY) { process.stdin.setRawMode(true); process.stdin.resume(); process.stdin.setEncoding('utf8'); diff --git a/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts b/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts index c694ab2940..418ac96709 100644 --- a/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts +++ b/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts @@ -5,14 +5,14 @@ import { CLOUD_TEMP_REPORTABLE_THRESHOLDS, ConcurrencyControlService, } from '@/concurrency/concurrency-control.service'; -import type { Logger } from '@/Logger'; +import type { Logger } from '@/logger'; import { InvalidConcurrencyLimitError } from '@/errors/invalid-concurrency-limit.error'; import { ConcurrencyQueue } from '../concurrency-queue'; import type { WorkflowExecuteMode as ExecutionMode } from 'n8n-workflow'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import type { IExecutingWorkflowData } from '@/Interfaces'; import type { Telemetry } from '@/telemetry'; -import type { EventService } from '@/eventbus/event.service'; +import type { EventService } from '@/events/event.service'; describe('ConcurrencyControlService', () => { const logger = mock(); diff --git a/packages/cli/src/concurrency/concurrency-control.service.ts b/packages/cli/src/concurrency/concurrency-control.service.ts index cf249c5172..b2b94a5d6e 100644 --- a/packages/cli/src/concurrency/concurrency-control.service.ts +++ b/packages/cli/src/concurrency/concurrency-control.service.ts @@ -1,4 +1,4 @@ -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import config from '@/config'; import { Service } from 'typedi'; import { ConcurrencyQueue } from './concurrency-queue'; @@ -8,7 +8,7 @@ import { ExecutionRepository } from '@/databases/repositories/execution.reposito import type { WorkflowExecuteMode as ExecutionMode } from 'n8n-workflow'; import type { IExecutingWorkflowData } from '@/Interfaces'; import { Telemetry } from '@/telemetry'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; export const CLOUD_TEMP_PRODUCTION_LIMIT = 999; export const CLOUD_TEMP_REPORTABLE_THRESHOLDS = [5, 10, 20, 50, 100, 200]; @@ -53,20 +53,20 @@ export class ConcurrencyControlService { this.isEnabled = true; - this.productionQueue.on('concurrency-check', ({ capacity }: { capacity: number }) => { + this.productionQueue.on('concurrency-check', ({ capacity }) => { if (this.shouldReport(capacity)) { - void this.telemetry.track('User hit concurrency limit', { + this.telemetry.track('User hit concurrency limit', { threshold: CLOUD_TEMP_PRODUCTION_LIMIT - capacity, }); } }); - this.productionQueue.on('execution-throttled', ({ executionId }: { executionId: string }) => { + this.productionQueue.on('execution-throttled', ({ executionId }) => { this.log('Execution throttled', { executionId }); this.eventService.emit('execution-throttled', { executionId }); }); - this.productionQueue.on('execution-released', async (executionId: string) => { + this.productionQueue.on('execution-released', async (executionId) => { this.log('Execution released', { executionId }); await this.executionRepository.resetStartedAt(executionId); }); diff --git a/packages/cli/src/concurrency/concurrency-queue.ts b/packages/cli/src/concurrency/concurrency-queue.ts index 90c62d2efc..1b578b5a8e 100644 --- a/packages/cli/src/concurrency/concurrency-queue.ts +++ b/packages/cli/src/concurrency/concurrency-queue.ts @@ -1,9 +1,14 @@ import { Service } from 'typedi'; -import { EventEmitter } from 'node:events'; -import debounce from 'lodash/debounce'; +import { TypedEmitter } from '@/TypedEmitter'; + +type ConcurrencyEvents = { + 'execution-throttled': { executionId: string }; + 'execution-released': string; + 'concurrency-check': { capacity: number }; +}; @Service() -export class ConcurrencyQueue extends EventEmitter { +export class ConcurrencyQueue extends TypedEmitter { private readonly queue: Array<{ executionId: string; resolve: () => void; @@ -63,9 +68,4 @@ export class ConcurrencyQueue extends EventEmitter { resolve(); } - - private debouncedEmit = debounce( - (event: string, payload: object) => this.emit(event, payload), - 300, - ); } diff --git a/packages/cli/test/unit/config/index.test.ts b/packages/cli/src/config/__tests__/index.test.ts similarity index 100% rename from packages/cli/test/unit/config/index.test.ts rename to packages/cli/src/config/__tests__/index.test.ts diff --git a/packages/cli/src/config/index.ts b/packages/cli/src/config/index.ts index f743e7961d..dedc803839 100644 --- a/packages/cli/src/config/index.ts +++ b/packages/cli/src/config/index.ts @@ -96,9 +96,10 @@ config.validate({ }); const userManagement = config.get('userManagement'); if (userManagement.jwtRefreshTimeoutHours >= userManagement.jwtSessionDurationHours) { - console.warn( - 'N8N_USER_MANAGEMENT_JWT_REFRESH_TIMEOUT_HOURS needs to smaller than N8N_USER_MANAGEMENT_JWT_DURATION_HOURS. Setting N8N_USER_MANAGEMENT_JWT_REFRESH_TIMEOUT_HOURS to 0 for now.', - ); + if (!inTest) + console.warn( + 'N8N_USER_MANAGEMENT_JWT_REFRESH_TIMEOUT_HOURS needs to smaller than N8N_USER_MANAGEMENT_JWT_DURATION_HOURS. Setting N8N_USER_MANAGEMENT_JWT_REFRESH_TIMEOUT_HOURS to 0 for now.', + ); config.set('userManagement.jwtRefreshTimeoutHours', 0); } diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 061ab08b36..d3334d4ab1 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -4,6 +4,7 @@ import { Container } from 'typedi'; import { InstanceSettings } from 'n8n-core'; import { LOG_LEVELS } from 'n8n-workflow'; import { ensureStringArray } from './utils'; +import { GlobalConfig } from '@n8n/config'; convict.addFormat({ name: 'comma-separated-list', @@ -161,119 +162,6 @@ export const schema = { }, }, - queue: { - health: { - active: { - doc: 'If health checks should be enabled', - format: Boolean, - default: false, - env: 'QUEUE_HEALTH_CHECK_ACTIVE', - }, - port: { - doc: 'Port to serve health check on if activated', - format: Number, - default: 5678, - env: 'QUEUE_HEALTH_CHECK_PORT', - }, - }, - bull: { - prefix: { - doc: 'Prefix for all bull queue keys', - format: String, - default: 'bull', - env: 'QUEUE_BULL_PREFIX', - }, - redis: { - db: { - doc: 'Redis DB', - format: Number, - default: 0, - env: 'QUEUE_BULL_REDIS_DB', - }, - host: { - doc: 'Redis Host', - format: String, - default: 'localhost', - env: 'QUEUE_BULL_REDIS_HOST', - }, - password: { - doc: 'Redis Password', - format: String, - default: '', - env: 'QUEUE_BULL_REDIS_PASSWORD', - }, - port: { - doc: 'Redis Port', - format: Number, - default: 6379, - env: 'QUEUE_BULL_REDIS_PORT', - }, - timeoutThreshold: { - doc: 'Max cumulative timeout (in milliseconds) of connection retries before process exit', - format: Number, - default: 10000, - env: 'QUEUE_BULL_REDIS_TIMEOUT_THRESHOLD', - }, - username: { - doc: 'Redis Username (needs Redis >= 6)', - format: String, - default: '', - env: 'QUEUE_BULL_REDIS_USERNAME', - }, - clusterNodes: { - doc: 'Redis Cluster startup nodes (comma separated list of host:port pairs)', - format: String, - default: '', - env: 'QUEUE_BULL_REDIS_CLUSTER_NODES', - }, - tls: { - format: Boolean, - default: false, - env: 'QUEUE_BULL_REDIS_TLS', - doc: 'Enable TLS on Redis connections. Default: false', - }, - }, - queueRecoveryInterval: { - doc: 'If > 0 enables an active polling to the queue that can recover for Redis crashes. Given in seconds; 0 is disabled. May increase Redis traffic significantly.', - format: Number, - default: 60, - env: 'QUEUE_RECOVERY_INTERVAL', - }, - gracefulShutdownTimeout: { - doc: '[DEPRECATED] (Use N8N_GRACEFUL_SHUTDOWN_TIMEOUT instead) How long should n8n wait for running executions before exiting worker process (seconds)', - format: Number, - default: 30, - env: 'QUEUE_WORKER_TIMEOUT', - }, - settings: { - lockDuration: { - doc: 'How long (ms) is the lease period for a worker to work on a message', - format: Number, - default: 30000, - env: 'QUEUE_WORKER_LOCK_DURATION', - }, - lockRenewTime: { - doc: 'How frequently (ms) should a worker renew the lease time', - format: Number, - default: 15000, - env: 'QUEUE_WORKER_LOCK_RENEW_TIME', - }, - stalledInterval: { - doc: 'How often check for stalled jobs (use 0 for never checking)', - format: Number, - default: 30000, - env: 'QUEUE_WORKER_STALLED_INTERVAL', - }, - maxStalledCount: { - doc: 'Max amount of times a stalled job will be re-processed', - format: Number, - default: 1, - env: 'QUEUE_WORKER_MAX_STALLED_COUNT', - }, - }, - }, - }, - generic: { // The timezone to use. Is important for nodes like "Cron" which start the // workflow automatically at a specified time. This setting can also be @@ -307,40 +195,6 @@ export const schema = { }, }, - // How n8n can be reached (Editor & REST-API) - path: { - format: String, - default: '/', - arg: 'path', - env: 'N8N_PATH', - doc: 'Path n8n is deployed to', - }, - host: { - format: String, - default: 'localhost', - arg: 'host', - env: 'N8N_HOST', - doc: 'Host name n8n can be reached', - }, - port: { - format: Number, - default: 5678, - arg: 'port', - env: 'N8N_PORT', - doc: 'HTTP port n8n can be reached', - }, - listen_address: { - format: String, - default: '0.0.0.0', - env: 'N8N_LISTEN_ADDRESS', - doc: 'IP address n8n should listen on', - }, - protocol: { - format: ['http', 'https'] as const, - default: 'http', - env: 'N8N_PROTOCOL', - doc: 'HTTP Protocol via which n8n can be reached', - }, secure_cookie: { doc: 'This sets the `Secure` flag on n8n auth cookie', format: Boolean, @@ -374,7 +228,7 @@ export const schema = { env: 'N8N_RESTRICT_FILE_ACCESS_TO', }, blockFileAccessToN8nFiles: { - doc: 'If set to true it will block access to all files in the ".n8n" directory and user defined config files.', + doc: 'If set to true it will block access to all files in the ".n8n" directory, the static cache dir at ~/.cache/n8n/public, and user defined config files.', format: Boolean, default: true, env: 'N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES', @@ -389,149 +243,6 @@ export const schema = { }, }, - endpoints: { - payloadSizeMax: { - format: Number, - default: 16, - env: 'N8N_PAYLOAD_SIZE_MAX', - doc: 'Maximum payload size in MB.', - }, - metrics: { - enable: { - format: Boolean, - default: false, - env: 'N8N_METRICS', - doc: 'Enable /metrics endpoint. Default: false', - }, - prefix: { - format: String, - default: 'n8n_', - env: 'N8N_METRICS_PREFIX', - doc: 'An optional prefix for metric names. Default: n8n_', - }, - includeDefaultMetrics: { - format: Boolean, - default: true, - env: 'N8N_METRICS_INCLUDE_DEFAULT_METRICS', - doc: 'Whether to expose default system and node.js metrics. Default: true', - }, - includeWorkflowIdLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL', - doc: 'Whether to include a label for the workflow ID on workflow metrics. Default: false', - }, - includeNodeTypeLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_NODE_TYPE_LABEL', - doc: 'Whether to include a label for the node type on node metrics. Default: false', - }, - includeCredentialTypeLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_CREDENTIAL_TYPE_LABEL', - doc: 'Whether to include a label for the credential type on credential metrics. Default: false', - }, - includeApiEndpoints: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_API_ENDPOINTS', - doc: 'Whether to expose metrics for API endpoints. Default: false', - }, - includeApiPathLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_API_PATH_LABEL', - doc: 'Whether to include a label for the path of API invocations. Default: false', - }, - includeApiMethodLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_API_METHOD_LABEL', - doc: 'Whether to include a label for the HTTP method (GET, POST, ...) of API invocations. Default: false', - }, - includeApiStatusCodeLabel: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_API_STATUS_CODE_LABEL', - doc: 'Whether to include a label for the HTTP status code (200, 404, ...) of API invocations. Default: false', - }, - includeCacheMetrics: { - format: Boolean, - default: false, - env: 'N8N_METRICS_INCLUDE_CACHE_METRICS', - doc: 'Whether to include metrics for cache hits and misses. Default: false', - }, - includeMessageEventBusMetrics: { - format: Boolean, - default: true, - env: 'N8N_METRICS_INCLUDE_MESSAGE_EVENT_BUS_METRICS', - doc: 'Whether to include metrics for events. Default: false', - }, - }, - rest: { - format: String, - default: 'rest', - env: 'N8N_ENDPOINT_REST', - doc: 'Path for rest endpoint', - }, - form: { - format: String, - default: 'form', - env: 'N8N_ENDPOINT_FORM', - doc: 'Path for form endpoint', - }, - formTest: { - format: String, - default: 'form-test', - env: 'N8N_ENDPOINT_FORM_TEST', - doc: 'Path for test form endpoint', - }, - formWaiting: { - format: String, - default: 'form-waiting', - env: 'N8N_ENDPOINT_FORM_WAIT', - doc: 'Path for waiting form endpoint', - }, - webhook: { - format: String, - default: 'webhook', - env: 'N8N_ENDPOINT_WEBHOOK', - doc: 'Path for webhook endpoint', - }, - webhookWaiting: { - format: String, - default: 'webhook-waiting', - env: 'N8N_ENDPOINT_WEBHOOK_WAIT', - doc: 'Path for waiting-webhook endpoint', - }, - webhookTest: { - format: String, - default: 'webhook-test', - env: 'N8N_ENDPOINT_WEBHOOK_TEST', - doc: 'Path for test-webhook endpoint', - }, - disableUi: { - format: Boolean, - default: false, - env: 'N8N_DISABLE_UI', - doc: 'Disable N8N UI (Frontend).', - }, - disableProductionWebhooksOnMainProcess: { - format: Boolean, - default: false, - env: 'N8N_DISABLE_PRODUCTION_MAIN_PROCESS', - doc: 'Disable production webhooks from main process. This helps ensures no http traffic load to main process when using webhook-specific processes.', - }, - additionalNonUIRoutes: { - doc: 'Additional endpoints to not open the UI on. Multiple endpoints can be separated by colon (":")', - format: String, - default: '', - env: 'N8N_ADDITIONAL_NON_UI_ROUTES', - }, - }, - workflowTagsDisabled: { format: Boolean, default: false, @@ -558,12 +269,17 @@ export const schema = { default: 0, env: 'N8N_USER_MANAGEMENT_JWT_REFRESH_TIMEOUT_HOURS', }, + + /** + * @important Do not remove until after cloud hooks are updated to stop using convict config. + */ isInstanceOwnerSetUp: { // n8n loads this setting from DB on startup doc: "Whether the instance owner's account has been set up", format: Boolean, default: false, }, + authenticationMethod: { doc: 'How to authenticate users (e.g. "email", "ldap", "saml")', format: ['email', 'ldap', 'saml'] as const, @@ -831,43 +547,19 @@ export const schema = { }, }, - cache: { - backend: { - doc: 'Backend to use for caching', - format: ['memory', 'redis', 'auto'] as const, - default: 'auto', - env: 'N8N_CACHE_BACKEND', - }, - memory: { - maxSize: { - doc: 'Maximum size of memory cache in bytes', - format: Number, - default: 3 * 1024 * 1024, // 3 MB - env: 'N8N_CACHE_MEMORY_MAX_SIZE', - }, - ttl: { - doc: 'Time to live for cached items in memory (in ms)', - format: Number, - default: 3600 * 1000, // 1 hour - env: 'N8N_CACHE_MEMORY_TTL', - }, - }, - redis: { - prefix: { - doc: 'Prefix for all cache keys', - format: String, - default: 'cache', - env: 'N8N_CACHE_REDIS_KEY_PREFIX', - }, - ttl: { - doc: 'Time to live for cached items in redis (in ms), 0 for no TTL', - format: Number, - default: 3600 * 1000, // 1 hour - env: 'N8N_CACHE_REDIS_TTL', - }, + /** + * @important Do not remove until after cloud hooks are updated to stop using convict config. + */ + endpoints: { + rest: { + format: String, + default: Container.get(GlobalConfig).endpoints.rest, }, }, + /** + * @important Do not remove until after cloud hooks are updated to stop using convict config. + */ ai: { enabled: { doc: 'Whether AI features are enabled', @@ -877,6 +569,15 @@ export const schema = { }, }, + aiAssistant: { + baseUrl: { + doc: 'Base URL of the AI assistant service', + format: String, + default: '', + env: 'N8N_AI_ASSISTANT_BASE_URL', + }, + }, + expression: { evaluator: { doc: 'Expression evaluator to use', @@ -918,11 +619,6 @@ export const schema = { }, multiMainSetup: { - instanceType: { - doc: 'Type of instance in multi-main setup', - format: ['unset', 'leader', 'follower'] as const, - default: 'unset', // only until first leader key check - }, enabled: { doc: 'Whether to enable multi-main setup for queue mode (license required)', format: Boolean, diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 41c95cb01a..045d5a30e1 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -90,6 +90,8 @@ export const LICENSE_FEATURES = { PROJECT_ROLE_ADMIN: 'feat:projectRole:admin', PROJECT_ROLE_EDITOR: 'feat:projectRole:editor', PROJECT_ROLE_VIEWER: 'feat:projectRole:viewer', + AI_ASSISTANT: 'feat:aiAssistant', + COMMUNITY_NODES_CUSTOM_REGISTRY: 'feat:communityNodes:customRegistry', } as const; export const LICENSE_QUOTAS = { diff --git a/packages/cli/test/unit/controllers/curl.controller.test.ts b/packages/cli/src/controllers/__tests__/curl.controller.test.ts similarity index 100% rename from packages/cli/test/unit/controllers/curl.controller.test.ts rename to packages/cli/src/controllers/__tests__/curl.controller.test.ts diff --git a/packages/cli/test/unit/controllers/dynamic-node-parameters.controller.test.ts b/packages/cli/src/controllers/__tests__/dynamic-node-parameters.controller.test.ts similarity index 94% rename from packages/cli/test/unit/controllers/dynamic-node-parameters.controller.test.ts rename to packages/cli/src/controllers/__tests__/dynamic-node-parameters.controller.test.ts index fe1313a4ec..0a129eb821 100644 --- a/packages/cli/test/unit/controllers/dynamic-node-parameters.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/dynamic-node-parameters.controller.test.ts @@ -2,7 +2,7 @@ import { DynamicNodeParametersController } from '@/controllers/dynamicNodeParame import type { DynamicNodeParametersRequest } from '@/requests'; import type { DynamicNodeParametersService } from '@/services/dynamicNodeParameters.service'; import { mock } from 'jest-mock-extended'; -import * as AdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as AdditionalData from '@/workflow-execute-additional-data'; import type { ILoadOptions, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; describe('DynamicNodeParametersController', () => { diff --git a/packages/cli/src/controllers/__tests__/me.controller.test.ts b/packages/cli/src/controllers/__tests__/me.controller.test.ts new file mode 100644 index 0000000000..99f4714023 --- /dev/null +++ b/packages/cli/src/controllers/__tests__/me.controller.test.ts @@ -0,0 +1,433 @@ +import type { Response } from 'express'; +import { Container } from 'typedi'; +import jwt from 'jsonwebtoken'; +import { mock, anyObject } from 'jest-mock-extended'; + +import type { PublicUser } from '@/Interfaces'; +import type { User } from '@db/entities/User'; +import { API_KEY_PREFIX, MeController } from '@/controllers/me.controller'; +import { AUTH_COOKIE_NAME } from '@/constants'; +import type { AuthenticatedRequest, MeRequest } from '@/requests'; +import { UserService } from '@/services/user.service'; +import { ExternalHooks } from '@/external-hooks'; +import { License } from '@/license'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import { EventService } from '@/events/event.service'; +import { badPasswords } from '@test/testData'; +import { mockInstance } from '@test/mocking'; +import { AuthUserRepository } from '@/databases/repositories/authUser.repository'; +import { InvalidAuthTokenRepository } from '@db/repositories/invalidAuthToken.repository'; +import { UserRepository } from '@db/repositories/user.repository'; +import { MfaService } from '@/mfa/mfa.service'; +import { InvalidMfaCodeError } from '@/errors/response-errors/invalid-mfa-code.error'; + +const browserId = 'test-browser-id'; + +describe('MeController', () => { + const externalHooks = mockInstance(ExternalHooks); + const eventService = mockInstance(EventService); + const userService = mockInstance(UserService); + const userRepository = mockInstance(UserRepository); + const mockMfaService = mockInstance(MfaService); + mockInstance(AuthUserRepository); + mockInstance(InvalidAuthTokenRepository); + mockInstance(License).isWithinUsersLimit.mockReturnValue(true); + const controller = Container.get(MeController); + + describe('updateCurrentUser', () => { + it('should throw BadRequestError if email is missing in the payload', async () => { + const req = mock({}); + await expect(controller.updateCurrentUser(req, mock())).rejects.toThrowError( + new BadRequestError('Email is mandatory'), + ); + }); + + it('should throw BadRequestError if email is invalid', async () => { + const req = mock({ body: { email: 'invalid-email' } }); + await expect(controller.updateCurrentUser(req, mock())).rejects.toThrowError( + new BadRequestError('Invalid email address'), + ); + }); + + it('should update the user in the DB, and issue a new cookie', async () => { + const user = mock({ + id: '123', + email: 'valid@email.com', + password: 'password', + authIdentities: [], + role: 'global:owner', + mfaEnabled: false, + }); + const req = mock({ user, browserId }); + req.body = { + email: 'valid@email.com', + firstName: 'John', + lastName: 'Potato', + }; + const res = mock(); + userRepository.findOneByOrFail.mockResolvedValue(user); + userRepository.findOneOrFail.mockResolvedValue(user); + jest.spyOn(jwt, 'sign').mockImplementation(() => 'signed-token'); + userService.toPublic.mockResolvedValue({} as unknown as PublicUser); + + await controller.updateCurrentUser(req, res); + + expect(externalHooks.run).toHaveBeenCalledWith('user.profile.beforeUpdate', [ + user.id, + user.email, + req.body, + ]); + + expect(userService.update).toHaveBeenCalled(); + expect(eventService.emit).toHaveBeenCalledWith('user-updated', { + user, + fieldsChanged: ['firstName', 'lastName'], // email did not change + }); + expect(res.cookie).toHaveBeenCalledWith( + AUTH_COOKIE_NAME, + 'signed-token', + expect.objectContaining({ + maxAge: expect.any(Number), + httpOnly: true, + sameSite: 'lax', + secure: false, + }), + ); + + expect(externalHooks.run).toHaveBeenCalledWith('user.profile.update', [ + user.email, + anyObject(), + ]); + }); + + it('should not allow updating any other fields on a user besides email and name', async () => { + const user = mock({ + id: '123', + password: 'password', + authIdentities: [], + role: 'global:member', + mfaEnabled: false, + }); + const req = mock({ user, browserId }); + req.body = { email: 'valid@email.com', firstName: 'John', lastName: 'Potato' }; + const res = mock(); + userRepository.findOneOrFail.mockResolvedValue(user); + jest.spyOn(jwt, 'sign').mockImplementation(() => 'signed-token'); + + // Add invalid data to the request payload + Object.assign(req.body, { id: '0', role: 'global:owner' }); + + await controller.updateCurrentUser(req, res); + + expect(userService.update).toHaveBeenCalled(); + + const updatePayload = userService.update.mock.calls[0][1]; + expect(updatePayload.email).toBe(req.body.email); + expect(updatePayload.firstName).toBe(req.body.firstName); + expect(updatePayload.lastName).toBe(req.body.lastName); + expect(updatePayload.id).toBeUndefined(); + expect(updatePayload.role).toBeUndefined(); + }); + + it('should throw BadRequestError if beforeUpdate hook throws BadRequestError', async () => { + const user = mock({ + id: '123', + password: 'password', + authIdentities: [], + role: 'global:owner', + mfaEnabled: false, + }); + const reqBody = { email: 'valid@email.com', firstName: 'John', lastName: 'Potato' }; + const req = mock({ user, body: reqBody }); + req.body = reqBody; // We don't want the body to be a mock object + + externalHooks.run.mockImplementationOnce(async (hookName) => { + if (hookName === 'user.profile.beforeUpdate') { + throw new BadRequestError('Invalid email address'); + } + }); + + await expect(controller.updateCurrentUser(req, mock())).rejects.toThrowError( + new BadRequestError('Invalid email address'), + ); + }); + + describe('when mfa is enabled', () => { + it('should throw BadRequestError if mfa code is missing', async () => { + const user = mock({ + id: '123', + email: 'valid@email.com', + password: 'password', + authIdentities: [], + role: 'global:owner', + mfaEnabled: true, + }); + const req = mock({ user, browserId }); + req.body = { email: 'new@email.com', firstName: 'John', lastName: 'Potato' }; + + await expect(controller.updateCurrentUser(req, mock())).rejects.toThrowError( + new BadRequestError('Two-factor code is required to change email'), + ); + }); + + it('should throw InvalidMfaCodeError if mfa code is invalid', async () => { + const user = mock({ + id: '123', + email: 'valid@email.com', + password: 'password', + authIdentities: [], + role: 'global:owner', + mfaEnabled: true, + }); + const req = mock({ user, browserId }); + req.body = { + email: 'new@email.com', + firstName: 'John', + lastName: 'Potato', + mfaCode: 'invalid', + }; + mockMfaService.validateMfa.mockResolvedValue(false); + + await expect(controller.updateCurrentUser(req, mock())).rejects.toThrow( + InvalidMfaCodeError, + ); + }); + + it("should update the user's email if mfa code is valid", async () => { + const user = mock({ + id: '123', + email: 'valid@email.com', + password: 'password', + authIdentities: [], + role: 'global:owner', + mfaEnabled: true, + }); + const req = mock({ user, browserId }); + req.body = { + email: 'new@email.com', + firstName: 'John', + lastName: 'Potato', + mfaCode: '123456', + }; + const res = mock(); + userRepository.findOneByOrFail.mockResolvedValue(user); + userRepository.findOneOrFail.mockResolvedValue(user); + jest.spyOn(jwt, 'sign').mockImplementation(() => 'signed-token'); + userService.toPublic.mockResolvedValue({} as unknown as PublicUser); + mockMfaService.validateMfa.mockResolvedValue(true); + + const result = await controller.updateCurrentUser(req, res); + + expect(result).toEqual({}); + }); + }); + }); + + describe('updatePassword', () => { + const passwordHash = '$2a$10$ffitcKrHT.Ls.m9FfWrMrOod76aaI0ogKbc3S96Q320impWpCbgj6'; // Hashed 'old_password' + + it('should throw if the user does not have a password set', async () => { + const req = mock({ + user: mock({ password: undefined }), + body: { currentPassword: '', newPassword: '' }, + }); + await expect(controller.updatePassword(req, mock())).rejects.toThrowError( + new BadRequestError('Requesting user not set up.'), + ); + }); + + it("should throw if currentPassword does not match the user's password", async () => { + const req = mock({ + user: mock({ password: passwordHash }), + body: { currentPassword: 'not_old_password', newPassword: '' }, + }); + await expect(controller.updatePassword(req, mock())).rejects.toThrowError( + new BadRequestError('Provided current password is incorrect.'), + ); + }); + + describe('should throw if newPassword is not valid', () => { + Object.entries(badPasswords).forEach(([newPassword, errorMessage]) => { + it(newPassword, async () => { + const req = mock({ + user: mock({ password: passwordHash }), + body: { currentPassword: 'old_password', newPassword }, + browserId, + }); + await expect(controller.updatePassword(req, mock())).rejects.toThrowError( + new BadRequestError(errorMessage), + ); + }); + }); + }); + + it('should update the password in the DB, and issue a new cookie', async () => { + const req = mock({ + user: mock({ password: passwordHash, mfaEnabled: false }), + body: { currentPassword: 'old_password', newPassword: 'NewPassword123' }, + browserId, + }); + const res = mock(); + userRepository.save.calledWith(req.user).mockResolvedValue(req.user); + jest.spyOn(jwt, 'sign').mockImplementation(() => 'new-signed-token'); + + await controller.updatePassword(req, res); + + expect(req.user.password).not.toBe(passwordHash); + + expect(res.cookie).toHaveBeenCalledWith( + AUTH_COOKIE_NAME, + 'new-signed-token', + expect.objectContaining({ + maxAge: expect.any(Number), + httpOnly: true, + sameSite: 'lax', + secure: false, + }), + ); + + expect(externalHooks.run).toHaveBeenCalledWith('user.password.update', [ + req.user.email, + req.user.password, + ]); + + expect(eventService.emit).toHaveBeenCalledWith('user-updated', { + user: req.user, + fieldsChanged: ['password'], + }); + }); + + describe('mfa enabled', () => { + it('should throw BadRequestError if mfa code is missing', async () => { + const req = mock({ + user: mock({ password: passwordHash, mfaEnabled: true }), + body: { currentPassword: 'old_password', newPassword: 'NewPassword123' }, + }); + + await expect(controller.updatePassword(req, mock())).rejects.toThrowError( + new BadRequestError('Two-factor code is required to change password.'), + ); + }); + + it('should throw InvalidMfaCodeError if invalid mfa code is given', async () => { + const req = mock({ + user: mock({ password: passwordHash, mfaEnabled: true }), + body: { currentPassword: 'old_password', newPassword: 'NewPassword123', mfaCode: '123' }, + }); + mockMfaService.validateMfa.mockResolvedValue(false); + + await expect(controller.updatePassword(req, mock())).rejects.toThrow(InvalidMfaCodeError); + }); + + it('should succeed when mfa code is correct', async () => { + const req = mock({ + user: mock({ password: passwordHash, mfaEnabled: true }), + body: { + currentPassword: 'old_password', + newPassword: 'NewPassword123', + mfaCode: 'valid', + }, + browserId, + }); + const res = mock(); + userRepository.save.calledWith(req.user).mockResolvedValue(req.user); + jest.spyOn(jwt, 'sign').mockImplementation(() => 'new-signed-token'); + mockMfaService.validateMfa.mockResolvedValue(true); + + const result = await controller.updatePassword(req, res); + + expect(result).toEqual({ success: true }); + expect(req.user.password).not.toBe(passwordHash); + }); + }); + }); + + describe('storeSurveyAnswers', () => { + it('should throw BadRequestError if answers are missing in the payload', async () => { + const req = mock({ + body: undefined, + }); + await expect(controller.storeSurveyAnswers(req)).rejects.toThrowError( + new BadRequestError('Personalization answers are mandatory'), + ); + }); + + test.each([ + 'automationGoalDevops', + 'companyIndustryExtended', + 'otherCompanyIndustryExtended', + 'automationGoalSm', + 'usageModes', + ])('should throw BadRequestError on XSS attempt for an array field %s', async (fieldName) => { + const req = mock(); + req.body = { + version: 'v4', + personalization_survey_n8n_version: '1.0.0', + personalization_survey_submitted_at: new Date().toISOString(), + [fieldName]: [''], + }; + + await expect(controller.storeSurveyAnswers(req)).rejects.toThrowError(BadRequestError); + }); + + test.each([ + 'automationGoalDevopsOther', + 'companySize', + 'companyType', + 'automationGoalSmOther', + 'roleOther', + 'reportedSource', + 'reportedSourceOther', + ])('should throw BadRequestError on XSS attempt for a string field %s', async (fieldName) => { + const req = mock(); + req.body = { + version: 'v4', + personalization_survey_n8n_version: '1.0.0', + personalization_survey_submitted_at: new Date().toISOString(), + [fieldName]: '', + }; + + await expect(controller.storeSurveyAnswers(req)).rejects.toThrowError(BadRequestError); + }); + }); + + describe('updateCurrentUserSettings', () => { + it('should throw BadRequestError on XSS attempt', async () => { + const req = mock({ + body: { + userActivated: '', + }, + }); + + await expect(controller.updateCurrentUserSettings(req)).rejects.toThrowError(BadRequestError); + }); + }); + + describe('API Key methods', () => { + let req: AuthenticatedRequest; + beforeAll(() => { + req = mock({ user: mock>({ id: '123', apiKey: `${API_KEY_PREFIX}test-key` }) }); + }); + + describe('createAPIKey', () => { + it('should create and save an API key', async () => { + const { apiKey } = await controller.createAPIKey(req); + expect(userService.update).toHaveBeenCalledWith(req.user.id, { apiKey }); + }); + }); + + describe('getAPIKey', () => { + it('should return the users api key redacted', async () => { + const { apiKey } = await controller.getAPIKey(req); + expect(apiKey).not.toEqual(req.user.apiKey); + }); + }); + + describe('deleteAPIKey', () => { + it('should delete the API key', async () => { + await controller.deleteAPIKey(req); + expect(userService.update).toHaveBeenCalledWith(req.user.id, { apiKey: null }); + }); + }); + }); +}); diff --git a/packages/cli/test/unit/controllers/owner.controller.test.ts b/packages/cli/src/controllers/__tests__/owner.controller.test.ts similarity index 93% rename from packages/cli/test/unit/controllers/owner.controller.test.ts rename to packages/cli/src/controllers/__tests__/owner.controller.test.ts index 0057c23708..0170333e8a 100644 --- a/packages/cli/test/unit/controllers/owner.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/owner.controller.test.ts @@ -10,18 +10,16 @@ import type { User } from '@db/entities/User'; import type { SettingsRepository } from '@db/repositories/settings.repository'; import type { UserRepository } from '@db/repositories/user.repository'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import type { InternalHooks } from '@/InternalHooks'; -import { License } from '@/License'; +import { License } from '@/license'; import type { OwnerRequest } from '@/requests'; import type { UserService } from '@/services/user.service'; import { PasswordUtility } from '@/services/password.utility'; -import { mockInstance } from '../../shared/mocking'; -import { badPasswords } from '../shared/testData'; +import { mockInstance } from '@test/mocking'; +import { badPasswords } from '@test/testData'; describe('OwnerController', () => { const configGetSpy = jest.spyOn(config, 'getEnv'); - const internalHooks = mock(); const authService = mock(); const userService = mock(); const userRepository = mock(); @@ -29,7 +27,7 @@ describe('OwnerController', () => { mockInstance(License).isWithinUsersLimit.mockReturnValue(true); const controller = new OwnerController( mock(), - internalHooks, + mock(), settingsRepository, authService, userService, diff --git a/packages/cli/test/unit/controllers/translation.controller.test.ts b/packages/cli/src/controllers/__tests__/translation.controller.test.ts similarity index 96% rename from packages/cli/test/unit/controllers/translation.controller.test.ts rename to packages/cli/src/controllers/__tests__/translation.controller.test.ts index e34237cd69..129a2f91a6 100644 --- a/packages/cli/test/unit/controllers/translation.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/translation.controller.test.ts @@ -6,7 +6,7 @@ import { CREDENTIAL_TRANSLATIONS_DIR, } from '@/controllers/translation.controller'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import type { CredentialTypes } from '@/CredentialTypes'; +import type { CredentialTypes } from '@/credential-types'; describe('TranslationController', () => { const configGetSpy = jest.spyOn(config, 'getEnv'); diff --git a/packages/cli/test/unit/controllers/userSettings.controller.test.ts b/packages/cli/src/controllers/__tests__/userSettings.controller.test.ts similarity index 96% rename from packages/cli/test/unit/controllers/userSettings.controller.test.ts rename to packages/cli/src/controllers/__tests__/userSettings.controller.test.ts index e29afb74cc..3794008641 100644 --- a/packages/cli/test/unit/controllers/userSettings.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/userSettings.controller.test.ts @@ -60,7 +60,7 @@ describe('UserSettingsController', () => { [], ], [ - 'updates user settings, reseting to waiting state', + 'updates user settings, resetting to waiting state', { waitingForResponse: true, ignoredCount: 0, @@ -137,7 +137,7 @@ describe('UserSettingsController', () => { 'is waitingForResponse but missing ignoredCount', { lastShownAt: 123, waitingForResponse: true }, ], - ])('thows error when request payload is %s', async (_, payload) => { + ])('throws error when request payload is %s', async (_, payload) => { const req = mock(); req.user.id = '1'; req.body = payload; diff --git a/packages/cli/src/controllers/__tests__/users.controller.test.ts b/packages/cli/src/controllers/__tests__/users.controller.test.ts new file mode 100644 index 0000000000..38a4399cab --- /dev/null +++ b/packages/cli/src/controllers/__tests__/users.controller.test.ts @@ -0,0 +1,52 @@ +import { mock } from 'jest-mock-extended'; +import { UsersController } from '../users.controller'; +import type { UserRequest } from '@/requests'; +import type { EventService } from '@/events/event.service'; +import type { User } from '@/databases/entities/User'; +import type { UserRepository } from '@/databases/repositories/user.repository'; +import type { ProjectService } from '@/services/project.service'; + +describe('UsersController', () => { + const eventService = mock(); + const userRepository = mock(); + const projectService = mock(); + const controller = new UsersController( + mock(), + mock(), + mock(), + mock(), + userRepository, + mock(), + mock(), + mock(), + mock(), + mock(), + projectService, + eventService, + ); + + beforeEach(() => { + jest.restoreAllMocks(); + }); + + describe('changeGlobalRole', () => { + it('should emit event user-changed-role', async () => { + const request = mock({ + user: { id: '123' }, + params: { id: '456' }, + body: { newRoleName: 'global:member' }, + }); + userRepository.findOne.mockResolvedValue(mock({ id: '456' })); + projectService.getUserOwnedOrAdminProjects.mockResolvedValue([]); + + await controller.changeGlobalRole(request); + + expect(eventService.emit).toHaveBeenCalledWith('user-changed-role', { + userId: '123', + targetUserId: '456', + targetUserNewRole: 'global:member', + publicApi: false, + }); + }); + }); +}); diff --git a/packages/cli/src/controllers/aiAssistant.controller.ts b/packages/cli/src/controllers/aiAssistant.controller.ts new file mode 100644 index 0000000000..473c20a79d --- /dev/null +++ b/packages/cli/src/controllers/aiAssistant.controller.ts @@ -0,0 +1,44 @@ +import { Post, RestController } from '@/decorators'; +import { AiAssistantService } from '@/services/aiAsisstant.service'; +import { AiAssistantRequest } from '@/requests'; +import { Response } from 'express'; +import type { AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; +import { Readable, promises } from 'node:stream'; +import { InternalServerError } from 'express-openapi-validator/dist/openapi.validator'; +import { strict as assert } from 'node:assert'; +import { ErrorReporterProxy } from 'n8n-workflow'; + +@RestController('/ai-assistant') +export class AiAssistantController { + constructor(private readonly aiAssistantService: AiAssistantService) {} + + @Post('/chat', { rateLimit: { limit: 100 } }) + async chat(req: AiAssistantRequest.Chat, res: Response) { + try { + const stream = await this.aiAssistantService.chat(req.body, req.user); + + if (stream.body) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + await promises.pipeline(Readable.fromWeb(stream.body), res); + } + } catch (e) { + // todo add sentry reporting + assert(e instanceof Error); + ErrorReporterProxy.error(e); + throw new InternalServerError({ message: `Something went wrong: ${e.message}` }); + } + } + + @Post('/chat/apply-suggestion') + async applySuggestion( + req: AiAssistantRequest.ApplySuggestion, + ): Promise { + try { + return await this.aiAssistantService.applySuggestion(req.body, req.user); + } catch (e) { + assert(e instanceof Error); + ErrorReporterProxy.error(e); + throw new InternalServerError({ message: `Something went wrong: ${e.message}` }); + } + } +} diff --git a/packages/cli/src/controllers/auth.controller.ts b/packages/cli/src/controllers/auth.controller.ts index 7994221d9d..fe48530831 100644 --- a/packages/cli/src/controllers/auth.controller.ts +++ b/packages/cli/src/controllers/auth.controller.ts @@ -1,9 +1,9 @@ import validator from 'validator'; +import { Response } from 'express'; import { AuthService } from '@/auth/auth.service'; import { Get, Post, RestController } from '@/decorators'; import { RESPONSE_ERROR_MESSAGES } from '@/constants'; -import { Request, Response } from 'express'; import type { User } from '@db/entities/User'; import { AuthenticatedRequest, LoginRequest, UserRequest } from '@/requests'; import type { PublicUser } from '@/Interfaces'; @@ -13,24 +13,22 @@ import { getCurrentAuthenticationMethod, isLdapCurrentAuthenticationMethod, isSamlCurrentAuthenticationMethod, -} from '@/sso/ssoHelpers'; -import { InternalHooks } from '../InternalHooks'; -import { License } from '@/License'; +} from '@/sso/sso-helpers'; +import { License } from '@/license'; import { UserService } from '@/services/user.service'; -import { MfaService } from '@/Mfa/mfa.service'; -import { Logger } from '@/Logger'; +import { MfaService } from '@/mfa/mfa.service'; +import { Logger } from '@/logger'; import { AuthError } from '@/errors/response-errors/auth.error'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { ApplicationError } from 'n8n-workflow'; import { UserRepository } from '@/databases/repositories/user.repository'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController() export class AuthController { constructor( private readonly logger: Logger, - private readonly internalHooks: InternalHooks, private readonly authService: AuthService, private readonly mfaService: MfaService, private readonly userService: UserService, @@ -179,7 +177,6 @@ export class AuthController { throw new BadRequestError('Invalid request'); } - void this.internalHooks.onUserInviteEmailClick({ inviter, invitee }); this.eventService.emit('user-invite-email-click', { inviter, invitee }); const { firstName, lastName } = inviter; @@ -188,7 +185,8 @@ export class AuthController { /** Log out a user */ @Post('/logout') - logout(_: Request, res: Response) { + async logout(req: AuthenticatedRequest, res: Response) { + await this.authService.invalidateToken(req); this.authService.clearCookie(res); return { loggedOut: true }; } diff --git a/packages/cli/src/controllers/communityPackages.controller.ts b/packages/cli/src/controllers/communityPackages.controller.ts index b37d3df249..d9e7f49719 100644 --- a/packages/cli/src/controllers/communityPackages.controller.ts +++ b/packages/cli/src/controllers/communityPackages.controller.ts @@ -1,20 +1,17 @@ -import { Request, Response, NextFunction } from 'express'; -import config from '@/config'; import { RESPONSE_ERROR_MESSAGES, STARTER_TEMPLATE_NAME, UNKNOWN_FAILURE_REASON, } from '@/constants'; -import { Delete, Get, Middleware, Patch, Post, RestController, GlobalScope } from '@/decorators'; +import { Delete, Get, Patch, Post, RestController, GlobalScope } from '@/decorators'; import { NodeRequest } from '@/requests'; import type { InstalledPackages } from '@db/entities/InstalledPackages'; import type { CommunityPackages } from '@/Interfaces'; -import { InternalHooks } from '@/InternalHooks'; import { Push } from '@/push'; import { CommunityPackagesService } from '@/services/communityPackages.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; const { PACKAGE_NOT_INSTALLED, @@ -37,22 +34,10 @@ export function isNpmError(error: unknown): error is { code: number; stdout: str export class CommunityPackagesController { constructor( private readonly push: Push, - private readonly internalHooks: InternalHooks, private readonly communityPackagesService: CommunityPackagesService, private readonly eventService: EventService, ) {} - // TODO: move this into a new decorator `@IfConfig('executions.mode', 'queue')` - @Middleware() - checkIfCommunityNodesEnabled(req: Request, res: Response, next: NextFunction) { - if (config.getEnv('executions.mode') === 'queue' && req.method !== 'GET') - res.status(400).json({ - status: 'error', - message: 'Package management is disabled when running in "queue" mode', - }); - else next(); - } - @Post('/') @GlobalScope('communityPackage:install') async installPackage(req: NodeRequest.Post) { @@ -101,7 +86,7 @@ export class CommunityPackagesController { let installedPackage: InstalledPackages; try { - installedPackage = await this.communityPackagesService.installNpmModule( + installedPackage = await this.communityPackagesService.installPackage( parsed.packageName, parsed.version, ); @@ -209,7 +194,7 @@ export class CommunityPackagesController { } try { - await this.communityPackagesService.removeNpmModule(name, installedPackage); + await this.communityPackagesService.removePackage(name, installedPackage); } catch (error) { const message = [ `Error removing package "${name}"`, @@ -254,7 +239,7 @@ export class CommunityPackagesController { } try { - const newInstalledPackage = await this.communityPackagesService.updateNpmModule( + const newInstalledPackage = await this.communityPackagesService.updatePackage( this.communityPackagesService.parseNpmPackageName(name).packageName, previouslyInstalledPackage, ); diff --git a/packages/cli/src/controllers/debug.controller.ts b/packages/cli/src/controllers/debug.controller.ts index 663c375611..3b340aad7e 100644 --- a/packages/cli/src/controllers/debug.controller.ts +++ b/packages/cli/src/controllers/debug.controller.ts @@ -1,5 +1,5 @@ import { Get, RestController } from '@/decorators'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; import { OrchestrationService } from '@/services/orchestration.service'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; diff --git a/packages/cli/src/controllers/dynamicNodeParameters.controller.ts b/packages/cli/src/controllers/dynamicNodeParameters.controller.ts index c6a46aa3d4..cff338e437 100644 --- a/packages/cli/src/controllers/dynamicNodeParameters.controller.ts +++ b/packages/cli/src/controllers/dynamicNodeParameters.controller.ts @@ -1,7 +1,7 @@ -import type { INodePropertyOptions } from 'n8n-workflow'; +import type { INodePropertyOptions, NodeParameterValueType } from 'n8n-workflow'; import { Post, RestController } from '@/decorators'; -import { getBase } from '@/WorkflowExecuteAdditionalData'; +import { getBase } from '@/workflow-execute-additional-data'; import { DynamicNodeParametersService } from '@/services/dynamicNodeParameters.service'; import { DynamicNodeParametersRequest } from '@/requests'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; @@ -92,4 +92,28 @@ export class DynamicNodeParametersController { credentials, ); } + + @Post('/action-result') + async getActionResult( + req: DynamicNodeParametersRequest.ActionResult, + ): Promise { + const { currentNodeParameters, nodeTypeAndVersion, path, credentials, handler, payload } = + req.body; + + const additionalData = await getBase(req.user.id, currentNodeParameters); + + if (handler) { + return await this.service.getActionResult( + handler, + path, + additionalData, + nodeTypeAndVersion, + currentNodeParameters, + payload, + credentials, + ); + } + + return; + } } diff --git a/packages/cli/src/controllers/e2e.controller.ts b/packages/cli/src/controllers/e2e.controller.ts index 3954bb1caf..6d8eb88fed 100644 --- a/packages/cli/src/controllers/e2e.controller.ts +++ b/packages/cli/src/controllers/e2e.controller.ts @@ -3,19 +3,19 @@ import { v4 as uuid } from 'uuid'; import config from '@/config'; import { SettingsRepository } from '@db/repositories/settings.repository'; import { UserRepository } from '@db/repositories/user.repository'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { License } from '@/License'; +import { License } from '@/license'; import { LICENSE_FEATURES, LICENSE_QUOTAS, UNLIMITED_LICENSE_QUOTA, inE2ETests } from '@/constants'; import { Patch, Post, RestController } from '@/decorators'; import type { UserSetupPayload } from '@/requests'; import type { BooleanLicenseFeature, IPushDataType, NumericLicenseFeature } from '@/Interfaces'; -import { MfaService } from '@/Mfa/mfa.service'; +import { MfaService } from '@/mfa/mfa.service'; import { Push } from '@/push'; import { CacheService } from '@/services/cache/cache.service'; import { PasswordUtility } from '@/services/password.utility'; import Container from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { AuthUserRepository } from '@/databases/repositories/authUser.repository'; if (!inE2ETests) { @@ -87,6 +87,8 @@ export class E2EController { [LICENSE_FEATURES.PROJECT_ROLE_ADMIN]: false, [LICENSE_FEATURES.PROJECT_ROLE_EDITOR]: false, [LICENSE_FEATURES.PROJECT_ROLE_VIEWER]: false, + [LICENSE_FEATURES.AI_ASSISTANT]: false, + [LICENSE_FEATURES.COMMUNITY_NODES_CUSTOM_REGISTRY]: false, }; private numericFeatures: Record = { diff --git a/packages/cli/src/controllers/invitation.controller.ts b/packages/cli/src/controllers/invitation.controller.ts index d89d077ae6..624cc9b3f0 100644 --- a/packages/cli/src/controllers/invitation.controller.ts +++ b/packages/cli/src/controllers/invitation.controller.ts @@ -6,25 +6,23 @@ import config from '@/config'; import { Post, GlobalScope, RestController } from '@/decorators'; import { RESPONSE_ERROR_MESSAGES } from '@/constants'; import { UserRequest } from '@/requests'; -import { License } from '@/License'; +import { License } from '@/license'; import { UserService } from '@/services/user.service'; -import { Logger } from '@/Logger'; -import { isSamlLicensedAndEnabled } from '@/sso/saml/samlHelpers'; +import { Logger } from '@/logger'; +import { isSamlLicensedAndEnabled } from '@/sso/saml/saml-helpers'; import { PasswordUtility } from '@/services/password.utility'; import { PostHogClient } from '@/posthog'; import type { User } from '@/databases/entities/User'; import { UserRepository } from '@db/repositories/user.repository'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; -import { InternalHooks } from '@/InternalHooks'; -import { ExternalHooks } from '@/ExternalHooks'; -import { EventService } from '@/eventbus/event.service'; +import { ExternalHooks } from '@/external-hooks'; +import { EventService } from '@/events/event.service'; @RestController('/invitations') export class InvitationController { constructor( private readonly logger: Logger, - private readonly internalHooks: InternalHooks, private readonly externalHooks: ExternalHooks, private readonly authService: AuthService, private readonly userService: UserService, @@ -168,11 +166,11 @@ export class InvitationController { this.authService.issueCookie(res, updatedUser, req.browserId); - void this.internalHooks.onUserSignup(updatedUser, { - user_type: 'email', - was_disabled_ldap_user: false, + this.eventService.emit('user-signed-up', { + user: updatedUser, + userType: 'email', + wasDisabledLdapUser: false, }); - this.eventService.emit('user-signed-up', { user: updatedUser }); const publicInvitee = await this.userService.toPublic(invitee); diff --git a/packages/cli/src/controllers/me.controller.ts b/packages/cli/src/controllers/me.controller.ts index 1e3fc22ca2..6cad9dcef5 100644 --- a/packages/cli/src/controllers/me.controller.ts +++ b/packages/cli/src/controllers/me.controller.ts @@ -6,7 +6,7 @@ import { randomBytes } from 'crypto'; import { AuthService } from '@/auth/auth.service'; import { Delete, Get, Patch, Post, RestController } from '@/decorators'; import { PasswordUtility } from '@/services/password.utility'; -import { validateEntity } from '@/GenericHelpers'; +import { validateEntity } from '@/generic-helpers'; import type { User } from '@db/entities/User'; import { AuthenticatedRequest, @@ -15,15 +15,19 @@ import { UserUpdatePayload, } from '@/requests'; import type { PublicUser } from '@/Interfaces'; -import { isSamlLicensedAndEnabled } from '@/sso/saml/samlHelpers'; +import { isSamlLicensedAndEnabled } from '@/sso/saml/saml-helpers'; import { UserService } from '@/services/user.service'; -import { Logger } from '@/Logger'; -import { ExternalHooks } from '@/ExternalHooks'; -import { InternalHooks } from '@/InternalHooks'; +import { Logger } from '@/logger'; +import { ExternalHooks } from '@/external-hooks'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { UserRepository } from '@/databases/repositories/user.repository'; import { isApiEnabled } from '@/PublicApi'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; +import { MfaService } from '@/mfa/mfa.service'; +import { InvalidMfaCodeError } from '@/errors/response-errors/invalid-mfa-code.error'; +import { PersonalizationSurveyAnswersV4 } from './survey-answers.dto'; + +export const API_KEY_PREFIX = 'n8n_api_'; export const isApiEnabledMiddleware: RequestHandler = (_, res, next) => { if (isApiEnabled()) { @@ -38,12 +42,12 @@ export class MeController { constructor( private readonly logger: Logger, private readonly externalHooks: ExternalHooks, - private readonly internalHooks: InternalHooks, private readonly authService: AuthService, private readonly userService: UserService, private readonly passwordUtility: PasswordUtility, private readonly userRepository: UserRepository, private readonly eventService: EventService, + private readonly mfaService: MfaService, ) {} /** @@ -51,7 +55,8 @@ export class MeController { */ @Patch('/') async updateCurrentUser(req: MeRequest.UserUpdate, res: Response): Promise { - const { id: userId, email: currentEmail } = req.user; + const { id: userId, email: currentEmail, mfaEnabled } = req.user; + const payload = plainToInstance(UserUpdatePayload, req.body, { excludeExtraneousValues: true }); const { email } = payload; @@ -73,22 +78,34 @@ export class MeController { await validateEntity(payload); + const isEmailBeingChanged = email !== currentEmail; + // If SAML is enabled, we don't allow the user to change their email address - if (isSamlLicensedAndEnabled()) { - if (email !== currentEmail) { - this.logger.debug( - 'Request to update user failed because SAML user may not change their email', - { - userId, - payload, - }, - ); - throw new BadRequestError('SAML user may not change their email'); + if (isSamlLicensedAndEnabled() && isEmailBeingChanged) { + this.logger.debug( + 'Request to update user failed because SAML user may not change their email', + { + userId, + payload, + }, + ); + throw new BadRequestError('SAML user may not change their email'); + } + + if (mfaEnabled && isEmailBeingChanged) { + if (!payload.mfaCode) { + throw new BadRequestError('Two-factor code is required to change email'); + } + + const isMfaTokenValid = await this.mfaService.validateMfa(userId, payload.mfaCode, undefined); + if (!isMfaTokenValid) { + throw new InvalidMfaCodeError(); } } await this.externalHooks.run('user.profile.beforeUpdate', [userId, currentEmail, payload]); + const preUpdateUser = await this.userRepository.findOneByOrFail({ id: userId }); await this.userService.update(userId, payload); const user = await this.userRepository.findOneOrFail({ where: { id: userId }, @@ -98,8 +115,11 @@ export class MeController { this.authService.issueCookie(res, user, req.browserId); - const fieldsChanged = Object.keys(payload); - void this.internalHooks.onUserUpdate({ user, fields_changed: fieldsChanged }); + const changeableFields = ['email', 'firstName', 'lastName'] as const; + const fieldsChanged = changeableFields.filter( + (key) => key in payload && payload[key] !== preUpdateUser[key], + ); + this.eventService.emit('user-updated', { user, fieldsChanged }); const publicUser = await this.userService.toPublic(user); @@ -112,12 +132,12 @@ export class MeController { /** * Update the logged-in user's password. */ - @Patch('/password') + @Patch('/password', { rateLimit: true }) async updatePassword(req: MeRequest.Password, res: Response) { const { user } = req; - const { currentPassword, newPassword } = req.body; + const { currentPassword, newPassword, mfaCode } = req.body; - // If SAML is enabled, we don't allow the user to change their email address + // If SAML is enabled, we don't allow the user to change their password if (isSamlLicensedAndEnabled()) { this.logger.debug('Attempted to change password for user, while SAML is enabled', { userId: user.id, @@ -142,6 +162,17 @@ export class MeController { const validPassword = this.passwordUtility.validate(newPassword); + if (user.mfaEnabled) { + if (typeof mfaCode !== 'string') { + throw new BadRequestError('Two-factor code is required to change password.'); + } + + const isMfaTokenValid = await this.mfaService.validateMfa(user.id, mfaCode, undefined); + if (!isMfaTokenValid) { + throw new InvalidMfaCodeError(); + } + } + user.password = await this.passwordUtility.hash(validPassword); const updatedUser = await this.userRepository.save(user, { transaction: false }); @@ -149,7 +180,6 @@ export class MeController { this.authService.issueCookie(res, updatedUser, req.browserId); - void this.internalHooks.onUserUpdate({ user: updatedUser, fields_changed: ['password'] }); this.eventService.emit('user-updated', { user: updatedUser, fieldsChanged: ['password'] }); await this.externalHooks.run('user.password.update', [updatedUser.email, updatedUser.password]); @@ -166,7 +196,7 @@ export class MeController { if (!personalizationAnswers) { this.logger.debug( - 'Request to store user personalization survey failed because of empty payload', + 'Request to store user personalization survey failed because of undefined payload', { userId: req.user.id, }, @@ -174,17 +204,28 @@ export class MeController { throw new BadRequestError('Personalization answers are mandatory'); } + const validatedAnswers = plainToInstance( + PersonalizationSurveyAnswersV4, + personalizationAnswers, + { excludeExtraneousValues: true }, + ); + + await validateEntity(validatedAnswers); + await this.userRepository.save( { id: req.user.id, - personalizationAnswers, + personalizationAnswers: validatedAnswers, }, { transaction: false }, ); this.logger.info('User survey updated successfully', { userId: req.user.id }); - void this.internalHooks.onPersonalizationSurveySubmitted(req.user.id, personalizationAnswers); + this.eventService.emit('user-submitted-personalization-survey', { + userId: req.user.id, + answers: validatedAnswers, + }); return { success: true }; } @@ -208,7 +249,8 @@ export class MeController { */ @Get('/api-key', { middlewares: [isApiEnabledMiddleware] }) async getAPIKey(req: AuthenticatedRequest) { - return { apiKey: req.user.apiKey }; + const apiKey = this.redactApiKey(req.user.apiKey); + return { apiKey }; } /** @@ -231,6 +273,9 @@ export class MeController { const payload = plainToInstance(UserSettingsUpdatePayload, req.body, { excludeExtraneousValues: true, }); + + await validateEntity(payload); + const { id } = req.user; await this.userService.updateSettings(id, payload); @@ -242,4 +287,14 @@ export class MeController { return user.settings; } + + private redactApiKey(apiKey: string | null) { + if (!apiKey) return; + const keepLength = 5; + return ( + API_KEY_PREFIX + + apiKey.slice(API_KEY_PREFIX.length, API_KEY_PREFIX.length + keepLength) + + '*'.repeat(apiKey.length - API_KEY_PREFIX.length - keepLength) + ); + } } diff --git a/packages/cli/src/controllers/mfa.controller.ts b/packages/cli/src/controllers/mfa.controller.ts index 3c58b944d5..3d10736a1a 100644 --- a/packages/cli/src/controllers/mfa.controller.ts +++ b/packages/cli/src/controllers/mfa.controller.ts @@ -1,6 +1,6 @@ -import { Delete, Get, Post, RestController } from '@/decorators'; +import { Get, Post, RestController } from '@/decorators'; import { AuthenticatedRequest, MFA } from '@/requests'; -import { MfaService } from '@/Mfa/mfa.service'; +import { MfaService } from '@/mfa/mfa.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; @RestController('/mfa') @@ -47,7 +47,7 @@ export class MFAController { }; } - @Post('/enable') + @Post('/enable', { rateLimit: true }) async activateMFA(req: MFA.Activate) { const { token = null } = req.body; const { id, mfaEnabled } = req.user; @@ -71,14 +71,19 @@ export class MFAController { await this.mfaService.enableMfa(id); } - @Delete('/disable') - async disableMFA(req: AuthenticatedRequest) { - const { id } = req.user; + @Post('/disable', { rateLimit: true }) + async disableMFA(req: MFA.Disable) { + const { id: userId } = req.user; + const { token = null } = req.body; - await this.mfaService.disableMfa(id); + if (typeof token !== 'string' || !token) { + throw new BadRequestError('Token is required to disable MFA feature'); + } + + await this.mfaService.disableMfa(userId, token); } - @Post('/verify') + @Post('/verify', { rateLimit: true }) async verifyMFA(req: MFA.Verify) { const { id } = req.user; const { token } = req.body; diff --git a/packages/cli/src/controllers/nodeTypes.controller.ts b/packages/cli/src/controllers/nodeTypes.controller.ts index fddefb7e10..122b9f1fbd 100644 --- a/packages/cli/src/controllers/nodeTypes.controller.ts +++ b/packages/cli/src/controllers/nodeTypes.controller.ts @@ -4,7 +4,7 @@ import { Request } from 'express'; import type { INodeTypeDescription, INodeTypeNameVersion } from 'n8n-workflow'; import { Post, RestController } from '@/decorators'; import config from '@/config'; -import { NodeTypes } from '@/NodeTypes'; +import { NodeTypes } from '@/node-types'; @RestController('/node-types') export class NodeTypesController { diff --git a/packages/cli/test/unit/controllers/oauth/oAuth1Credential.controller.test.ts b/packages/cli/src/controllers/oauth/__tests__/oAuth1Credential.controller.test.ts similarity index 96% rename from packages/cli/test/unit/controllers/oauth/oAuth1Credential.controller.test.ts rename to packages/cli/src/controllers/oauth/__tests__/oAuth1Credential.controller.test.ts index cbadc1cf87..d0031186f7 100644 --- a/packages/cli/test/unit/controllers/oauth/oAuth1Credential.controller.test.ts +++ b/packages/cli/src/controllers/oauth/__tests__/oAuth1Credential.controller.test.ts @@ -11,15 +11,15 @@ import type { User } from '@db/entities/User'; import type { OAuthRequest } from '@/requests'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; -import { ExternalHooks } from '@/ExternalHooks'; -import { Logger } from '@/Logger'; +import { ExternalHooks } from '@/external-hooks'; +import { Logger } from '@/logger'; import { VariablesService } from '@/environments/variables/variables.service.ee'; -import { SecretsHelper } from '@/SecretsHelpers'; -import { CredentialsHelper } from '@/CredentialsHelper'; +import { SecretsHelper } from '@/secrets-helpers'; +import { CredentialsHelper } from '@/credentials-helper'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import { mockInstance } from '../../../shared/mocking'; +import { mockInstance } from '@test/mocking'; describe('OAuth1CredentialController', () => { mockInstance(Logger); diff --git a/packages/cli/test/unit/controllers/oauth/oAuth2Credential.controller.test.ts b/packages/cli/src/controllers/oauth/__tests__/oAuth2Credential.controller.test.ts similarity index 96% rename from packages/cli/test/unit/controllers/oauth/oAuth2Credential.controller.test.ts rename to packages/cli/src/controllers/oauth/__tests__/oAuth2Credential.controller.test.ts index f2b718fda0..c81a0c1559 100644 --- a/packages/cli/test/unit/controllers/oauth/oAuth2Credential.controller.test.ts +++ b/packages/cli/src/controllers/oauth/__tests__/oAuth2Credential.controller.test.ts @@ -11,15 +11,15 @@ import type { User } from '@db/entities/User'; import type { OAuthRequest } from '@/requests'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; -import { ExternalHooks } from '@/ExternalHooks'; -import { Logger } from '@/Logger'; +import { ExternalHooks } from '@/external-hooks'; +import { Logger } from '@/logger'; import { VariablesService } from '@/environments/variables/variables.service.ee'; -import { SecretsHelper } from '@/SecretsHelpers'; -import { CredentialsHelper } from '@/CredentialsHelper'; +import { SecretsHelper } from '@/secrets-helpers'; +import { CredentialsHelper } from '@/credentials-helper'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import { mockInstance } from '../../../shared/mocking'; +import { mockInstance } from '@test/mocking'; describe('OAuth2CredentialController', () => { mockInstance(Logger); diff --git a/packages/cli/src/controllers/oauth/abstractOAuth.controller.ts b/packages/cli/src/controllers/oauth/abstractOAuth.controller.ts index 9454b4ee7d..0d14b222d4 100644 --- a/packages/cli/src/controllers/oauth/abstractOAuth.controller.ts +++ b/packages/cli/src/controllers/oauth/abstractOAuth.controller.ts @@ -5,21 +5,20 @@ import { Credentials } from 'n8n-core'; import type { ICredentialDataDecryptedObject, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; import { jsonParse, ApplicationError } from 'n8n-workflow'; -import config from '@/config'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; -import type { User } from '@db/entities/User'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import type { ICredentialsDb } from '@/Interfaces'; import type { OAuthRequest } from '@/requests'; import { RESPONSE_ERROR_MESSAGES } from '@/constants'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; -import { Logger } from '@/Logger'; -import { ExternalHooks } from '@/ExternalHooks'; +import { CredentialsHelper } from '@/credentials-helper'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import { Logger } from '@/logger'; +import { ExternalHooks } from '@/external-hooks'; import { UrlService } from '@/services/url.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import { GlobalConfig } from '@n8n/config'; export interface CsrfStateParam { cid: string; @@ -37,10 +36,11 @@ export abstract class AbstractOAuthController { private readonly credentialsRepository: CredentialsRepository, private readonly sharedCredentialsRepository: SharedCredentialsRepository, private readonly urlService: UrlService, + private readonly globalConfig: GlobalConfig, ) {} get baseUrl() { - const restUrl = `${this.urlService.getInstanceBaseUrl()}/${config.getEnv('endpoints.rest')}`; + const restUrl = `${this.urlService.getInstanceBaseUrl()}/${this.globalConfig.endpoints.rest}`; return `${restUrl}/oauth${this.oauthVersion}-credential`; } @@ -70,8 +70,8 @@ export abstract class AbstractOAuthController { return credential; } - protected async getAdditionalData(user: User) { - return await WorkflowExecuteAdditionalData.getBase(user.id); + protected async getAdditionalData() { + return await WorkflowExecuteAdditionalData.getBase(); } protected async getDecryptedData( @@ -118,7 +118,7 @@ export abstract class AbstractOAuthController { return await this.credentialsRepository.findOneBy({ id: credentialId }); } - protected createCsrfState(credentialsId: string): [string, string] { + createCsrfState(credentialsId: string): [string, string] { const token = new Csrf(); const csrfSecret = token.secretSync(); const state: CsrfStateParam = { diff --git a/packages/cli/src/controllers/oauth/oAuth1Credential.controller.ts b/packages/cli/src/controllers/oauth/oAuth1Credential.controller.ts index 578a209e36..21bbe83f99 100644 --- a/packages/cli/src/controllers/oauth/oAuth1Credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oAuth1Credential.controller.ts @@ -6,7 +6,7 @@ import clientOAuth1 from 'oauth-1.0a'; import { createHmac } from 'crypto'; import { Get, RestController } from '@/decorators'; import { OAuthRequest } from '@/requests'; -import { sendErrorResponse } from '@/ResponseHelper'; +import { sendErrorResponse } from '@/response-helper'; import { AbstractOAuthController, type CsrfStateParam } from './abstractOAuth.controller'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; @@ -33,7 +33,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { @Get('/auth') async getAuthUri(req: OAuthRequest.OAuth1Credential.Auth): Promise { const credential = await this.getCredential(req); - const additionalData = await this.getAdditionalData(req.user); + const additionalData = await this.getAdditionalData(); const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); const oauthCredentials = this.applyDefaultsAndOverwrites( credential, @@ -99,9 +99,8 @@ export class OAuth1CredentialController extends AbstractOAuthController { } /** Verify and store app code. Generate access tokens and store for respective credential */ - @Get('/callback', { usesTemplates: true }) + @Get('/callback', { usesTemplates: true, skipAuth: true }) async handleCallback(req: OAuthRequest.OAuth1Credential.Callback, res: Response) { - const userId = req.user?.id; try { const { oauth_verifier, oauth_token, state: encodedState } = req.query; @@ -124,11 +123,11 @@ export class OAuth1CredentialController extends AbstractOAuthController { const credential = await this.getCredentialWithoutUser(credentialId); if (!credential) { const errorMessage = 'OAuth1 callback failed because of insufficient permissions'; - this.logger.error(errorMessage, { userId, credentialId }); + this.logger.error(errorMessage, { credentialId }); return this.renderCallbackError(res, errorMessage); } - const additionalData = await this.getAdditionalData(req.user); + const additionalData = await this.getAdditionalData(); const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); const oauthCredentials = this.applyDefaultsAndOverwrites( credential, @@ -138,7 +137,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { if (this.verifyCsrfState(decryptedDataOriginal, state)) { const errorMessage = 'The OAuth1 callback state is invalid!'; - this.logger.debug(errorMessage, { userId, credentialId }); + this.logger.debug(errorMessage, { credentialId }); return this.renderCallbackError(res, errorMessage); } @@ -156,7 +155,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { try { oauthToken = await axios.request(options); } catch (error) { - this.logger.error('Unable to fetch tokens for OAuth1 callback', { userId, credentialId }); + this.logger.error('Unable to fetch tokens for OAuth1 callback', { credentialId }); const errorResponse = new NotFoundError('Unable to get access tokens!'); return sendErrorResponse(res, errorResponse); } @@ -172,15 +171,11 @@ export class OAuth1CredentialController extends AbstractOAuthController { await this.encryptAndSaveData(credential, decryptedDataOriginal); this.logger.verbose('OAuth1 callback successful for new credential', { - userId, credentialId, }); return res.render('oauth-callback'); } catch (error) { - this.logger.error('OAuth1 callback failed because of insufficient user permissions', { - userId, - }); - // Error response + this.logger.error('OAuth1 callback failed because of insufficient user permissions'); return sendErrorResponse(res, error as Error); } } diff --git a/packages/cli/src/controllers/oauth/oAuth2Credential.controller.ts b/packages/cli/src/controllers/oauth/oAuth2Credential.controller.ts index 71a0fe140c..5b7929495f 100644 --- a/packages/cli/src/controllers/oauth/oAuth2Credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oAuth2Credential.controller.ts @@ -20,7 +20,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { @Get('/auth') async getAuthUri(req: OAuthRequest.OAuth2Credential.Auth): Promise { const credential = await this.getCredential(req); - const additionalData = await this.getAdditionalData(req.user); + const additionalData = await this.getAdditionalData(); const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); // At some point in the past we saved hidden scopes to credentials (but shouldn't) @@ -80,9 +80,8 @@ export class OAuth2CredentialController extends AbstractOAuthController { } /** Verify and store app code. Generate access tokens and store for respective credential */ - @Get('/callback', { usesTemplates: true }) + @Get('/callback', { usesTemplates: true, skipAuth: true }) async handleCallback(req: OAuthRequest.OAuth2Credential.Callback, res: Response) { - const userId = req.user?.id; try { const { code, state: encodedState } = req.query; if (!code || !encodedState) { @@ -104,11 +103,11 @@ export class OAuth2CredentialController extends AbstractOAuthController { const credential = await this.getCredentialWithoutUser(credentialId); if (!credential) { const errorMessage = 'OAuth2 callback failed because of insufficient permissions'; - this.logger.error(errorMessage, { userId, credentialId }); + this.logger.error(errorMessage, { credentialId }); return this.renderCallbackError(res, errorMessage); } - const additionalData = await this.getAdditionalData(req.user); + const additionalData = await this.getAdditionalData(); const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); const oauthCredentials = this.applyDefaultsAndOverwrites( credential, @@ -118,7 +117,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { if (this.verifyCsrfState(decryptedDataOriginal, state)) { const errorMessage = 'The OAuth2 callback state is invalid!'; - this.logger.debug(errorMessage, { userId, credentialId }); + this.logger.debug(errorMessage, { credentialId }); return this.renderCallbackError(res, errorMessage); } @@ -157,7 +156,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { if (oauthToken === undefined) { const errorMessage = 'Unable to get OAuth2 access tokens!'; - this.logger.error(errorMessage, { userId, credentialId }); + this.logger.error(errorMessage, { credentialId }); return this.renderCallbackError(res, errorMessage); } @@ -174,7 +173,6 @@ export class OAuth2CredentialController extends AbstractOAuthController { await this.encryptAndSaveData(credential, decryptedDataOriginal); this.logger.verbose('OAuth2 callback successful for credential', { - userId, credentialId, }); diff --git a/packages/cli/src/controllers/orchestration.controller.ts b/packages/cli/src/controllers/orchestration.controller.ts index 74a9665e1e..852d6f7cc3 100644 --- a/packages/cli/src/controllers/orchestration.controller.ts +++ b/packages/cli/src/controllers/orchestration.controller.ts @@ -1,7 +1,7 @@ import { Post, RestController, GlobalScope } from '@/decorators'; import { OrchestrationRequest } from '@/requests'; import { OrchestrationService } from '@/services/orchestration.service'; -import { License } from '@/License'; +import { License } from '@/license'; @RestController('/orchestration') export class OrchestrationController { diff --git a/packages/cli/src/controllers/owner.controller.ts b/packages/cli/src/controllers/owner.controller.ts index 6077026c90..826bbc2fa7 100644 --- a/packages/cli/src/controllers/owner.controller.ts +++ b/packages/cli/src/controllers/owner.controller.ts @@ -3,7 +3,7 @@ import { Response } from 'express'; import { AuthService } from '@/auth/auth.service'; import config from '@/config'; -import { validateEntity } from '@/GenericHelpers'; +import { validateEntity } from '@/generic-helpers'; import { GlobalScope, Post, RestController } from '@/decorators'; import { PasswordUtility } from '@/services/password.utility'; import { OwnerRequest } from '@/requests'; @@ -11,15 +11,15 @@ import { SettingsRepository } from '@db/repositories/settings.repository'; import { UserRepository } from '@db/repositories/user.repository'; import { PostHogClient } from '@/posthog'; import { UserService } from '@/services/user.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { InternalHooks } from '@/InternalHooks'; +import { EventService } from '@/events/event.service'; @RestController('/owner') export class OwnerController { constructor( private readonly logger: Logger, - private readonly internalHooks: InternalHooks, + private readonly eventService: EventService, private readonly settingsRepository: SettingsRepository, private readonly authService: AuthService, private readonly userService: UserService, @@ -85,7 +85,7 @@ export class OwnerController { this.authService.issueCookie(res, owner, req.browserId); - void this.internalHooks.onInstanceOwnerSetup({ user_id: owner.id }); + this.eventService.emit('instance-owner-setup', { userId: owner.id }); return await this.userService.toPublic(owner, { posthog: this.postHog, withScopes: true }); } diff --git a/packages/cli/src/controllers/passwordReset.controller.ts b/packages/cli/src/controllers/passwordReset.controller.ts index c548e607fd..3f93ee08e9 100644 --- a/packages/cli/src/controllers/passwordReset.controller.ts +++ b/packages/cli/src/controllers/passwordReset.controller.ts @@ -4,16 +4,15 @@ import validator from 'validator'; import { AuthService } from '@/auth/auth.service'; import { Get, Post, RestController } from '@/decorators'; import { PasswordUtility } from '@/services/password.utility'; -import { UserManagementMailer } from '@/UserManagement/email'; +import { UserManagementMailer } from '@/user-management/email'; import { PasswordResetRequest } from '@/requests'; -import { isSamlCurrentAuthenticationMethod } from '@/sso/ssoHelpers'; +import { isSamlCurrentAuthenticationMethod } from '@/sso/sso-helpers'; import { UserService } from '@/services/user.service'; -import { License } from '@/License'; +import { License } from '@/license'; import { RESPONSE_ERROR_MESSAGES } from '@/constants'; -import { MfaService } from '@/Mfa/mfa.service'; -import { Logger } from '@/Logger'; -import { ExternalHooks } from '@/ExternalHooks'; -import { InternalHooks } from '@/InternalHooks'; +import { MfaService } from '@/mfa/mfa.service'; +import { Logger } from '@/logger'; +import { ExternalHooks } from '@/external-hooks'; import { UrlService } from '@/services/url.service'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; @@ -21,14 +20,13 @@ import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { UnprocessableRequestError } from '@/errors/response-errors/unprocessable.error'; import { UserRepository } from '@/databases/repositories/user.repository'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController() export class PasswordResetController { constructor( private readonly logger: Logger, private readonly externalHooks: ExternalHooks, - private readonly internalHooks: InternalHooks, private readonly mailer: UserManagementMailer, private readonly authService: AuthService, private readonly userService: UserService, @@ -120,25 +118,23 @@ export class PasswordResetController { domain: this.urlService.getInstanceBaseUrl(), }); } catch (error) { - void this.internalHooks.onEmailFailed({ + this.eventService.emit('email-failed', { user, - message_type: 'Reset password', - public_api: false, + messageType: 'Reset password', + publicApi: false, }); - this.eventService.emit('email-failed', { user, messageType: 'Reset password' }); if (error instanceof Error) { throw new InternalServerError(`Please contact your administrator: ${error.message}`); } } this.logger.info('Sent password reset email successfully', { userId: user.id, email }); - void this.internalHooks.onUserTransactionalEmail({ - user_id: id, - message_type: 'Reset password', - public_api: false, + this.eventService.emit('user-transactional-email-sent', { + userId: id, + messageType: 'Reset password', + publicApi: false, }); - void this.internalHooks.onUserPasswordResetRequestClick({ user }); this.eventService.emit('user-password-reset-request-click', { user }); } @@ -171,7 +167,6 @@ export class PasswordResetController { } this.logger.info('Reset-password token resolved successfully', { userId: user.id }); - void this.internalHooks.onUserPasswordResetEmailClick({ user }); this.eventService.emit('user-password-reset-email-click', { user }); } @@ -215,17 +210,16 @@ export class PasswordResetController { this.authService.issueCookie(res, user, req.browserId); - void this.internalHooks.onUserUpdate({ user, fields_changed: ['password'] }); this.eventService.emit('user-updated', { user, fieldsChanged: ['password'] }); - // if this user used to be an LDAP users + // if this user used to be an LDAP user const ldapIdentity = user?.authIdentities?.find((i) => i.providerType === 'ldap'); if (ldapIdentity) { - void this.internalHooks.onUserSignup(user, { - user_type: 'email', - was_disabled_ldap_user: true, + this.eventService.emit('user-signed-up', { + user, + userType: 'email', + wasDisabledLdapUser: true, }); - this.eventService.emit('user-signed-up', { user }); } await this.externalHooks.run('user.password.update', [user.email, passwordHash]); diff --git a/packages/cli/src/controllers/project.controller.ts b/packages/cli/src/controllers/project.controller.ts index 848ba4b84c..e93b919ecb 100644 --- a/packages/cli/src/controllers/project.controller.ts +++ b/packages/cli/src/controllers/project.controller.ts @@ -23,7 +23,7 @@ import { ProjectRepository } from '@/databases/repositories/project.repository'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In, Not } from '@n8n/typeorm'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController('/projects') export class ProjectController { diff --git a/packages/cli/src/controllers/survey-answers.dto.ts b/packages/cli/src/controllers/survey-answers.dto.ts new file mode 100644 index 0000000000..f115a6992b --- /dev/null +++ b/packages/cli/src/controllers/survey-answers.dto.ts @@ -0,0 +1,109 @@ +import { NoXss } from '@/validators/no-xss.validator'; +import { Expose } from 'class-transformer'; +import { IsString, IsArray, IsOptional, IsEmail, IsEnum } from 'class-validator'; +import type { IPersonalizationSurveyAnswersV4 } from 'n8n-workflow'; + +export class PersonalizationSurveyAnswersV4 implements IPersonalizationSurveyAnswersV4 { + @NoXss() + @Expose() + @IsEnum(['v4']) + version: 'v4'; + + @NoXss() + @Expose() + @IsString() + personalization_survey_submitted_at: string; + + @NoXss() + @Expose() + @IsString() + personalization_survey_n8n_version: string; + + @Expose() + @IsOptional() + @IsArray() + @NoXss({ each: true }) + @IsString({ each: true }) + automationGoalDevops?: string[] | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + automationGoalDevopsOther?: string | null; + + @NoXss({ each: true }) + @Expose() + @IsOptional() + @IsArray() + @IsString({ each: true }) + companyIndustryExtended?: string[] | null; + + @NoXss({ each: true }) + @Expose() + @IsOptional() + @IsString({ each: true }) + otherCompanyIndustryExtended?: string[] | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + companySize?: string | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + companyType?: string | null; + + @NoXss({ each: true }) + @Expose() + @IsOptional() + @IsArray() + @IsString({ each: true }) + automationGoalSm?: string[] | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + automationGoalSmOther?: string | null; + + @NoXss({ each: true }) + @Expose() + @IsOptional() + @IsArray() + @IsString({ each: true }) + usageModes?: string[] | null; + + @NoXss() + @Expose() + @IsOptional() + @IsEmail() + email?: string | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + role?: string | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + roleOther?: string | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + reportedSource?: string | null; + + @NoXss() + @Expose() + @IsOptional() + @IsString() + reportedSourceOther?: string | null; +} diff --git a/packages/cli/src/controllers/translation.controller.ts b/packages/cli/src/controllers/translation.controller.ts index f359ec2b3a..03a8c3b035 100644 --- a/packages/cli/src/controllers/translation.controller.ts +++ b/packages/cli/src/controllers/translation.controller.ts @@ -6,7 +6,7 @@ import config from '@/config'; import { NODES_BASE_DIR } from '@/constants'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; -import { CredentialTypes } from '@/CredentialTypes'; +import { CredentialTypes } from '@/credential-types'; export const CREDENTIAL_TRANSLATIONS_DIR = 'n8n-nodes-base/dist/credentials/translations'; export const NODE_HEADERS_PATH = join(NODES_BASE_DIR, 'dist/nodes/headers'); diff --git a/packages/cli/src/controllers/users.controller.ts b/packages/cli/src/controllers/users.controller.ts index 0fd8481ed8..43916903fe 100644 --- a/packages/cli/src/controllers/users.controller.ts +++ b/packages/cli/src/controllers/users.controller.ts @@ -9,33 +9,31 @@ import { UserRoleChangePayload, UserSettingsUpdatePayload, } from '@/requests'; -import type { PublicUser, ITelemetryUserDeletionData } from '@/Interfaces'; +import type { PublicUser } from '@/Interfaces'; import { AuthIdentity } from '@db/entities/AuthIdentity'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { UserRepository } from '@db/repositories/user.repository'; import { UserService } from '@/services/user.service'; import { listQueryMiddleware } from '@/middlewares'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { ExternalHooks } from '@/ExternalHooks'; -import { InternalHooks } from '@/InternalHooks'; -import { validateEntity } from '@/GenericHelpers'; +import { ExternalHooks } from '@/external-hooks'; +import { validateEntity } from '@/generic-helpers'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { Project } from '@/databases/entities/Project'; import { WorkflowService } from '@/workflows/workflow.service'; import { CredentialsService } from '@/credentials/credentials.service'; import { ProjectService } from '@/services/project.service'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController('/users') export class UsersController { constructor( private readonly logger: Logger, private readonly externalHooks: ExternalHooks, - private readonly internalHooks: InternalHooks, private readonly sharedCredentialsRepository: SharedCredentialsRepository, private readonly sharedWorkflowRepository: SharedWorkflowRepository, private readonly userRepository: UserRepository, @@ -183,12 +181,7 @@ export class UsersController { ); } - const telemetryData: ITelemetryUserDeletionData = { - user_id: req.user.id, - target_user_old_status: userToDelete.isPending ? 'invited' : 'active', - target_user_id: idToDelete, - migration_strategy: transferId ? 'transfer_data' : 'delete_data', - }; + let transfereeId; if (transferId) { const transfereePersonalProject = await this.projectRepository.findOneBy({ id: transferId }); @@ -206,7 +199,7 @@ export class UsersController { }, }); - telemetryData.migration_user_id = transferee.id; + transfereeId = transferee.id; await this.userService.getManager().transaction(async (trx) => { await this.workflowService.transferAll( @@ -253,12 +246,14 @@ export class UsersController { await trx.delete(User, { id: userToDelete.id }); }); - void this.internalHooks.onUserDeletion({ + this.eventService.emit('user-deleted', { user: req.user, - telemetryData, publicApi: false, + targetUserOldStatus: userToDelete.isPending ? 'invited' : 'active', + targetUserId: idToDelete, + migrationStrategy: transferId ? 'transfer_data' : 'delete_data', + migrationUserId: transfereeId, }); - this.eventService.emit('user-deleted', { user: req.user }); await this.externalHooks.run('user.deleted', [await this.userService.toPublic(userToDelete)]); @@ -294,11 +289,11 @@ export class UsersController { await this.userService.update(targetUser.id, { role: payload.newRoleName }); - void this.internalHooks.onUserRoleChange({ - user: req.user, - target_user_id: targetUser.id, - target_user_new_role: ['global', payload.newRoleName].join(' '), - public_api: false, + this.eventService.emit('user-changed-role', { + userId: req.user.id, + targetUserId: targetUser.id, + targetUserNewRole: payload.newRoleName, + publicApi: false, }); const projects = await this.projectService.getUserOwnedOrAdminProjects(targetUser.id); diff --git a/packages/cli/src/controllers/workflowStatistics.controller.ts b/packages/cli/src/controllers/workflowStatistics.controller.ts index 0c86786129..2cae52e8cc 100644 --- a/packages/cli/src/controllers/workflowStatistics.controller.ts +++ b/packages/cli/src/controllers/workflowStatistics.controller.ts @@ -5,7 +5,7 @@ import { StatisticsNames } from '@db/entities/WorkflowStatistics'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { WorkflowStatisticsRepository } from '@db/repositories/workflowStatistics.repository'; import type { IWorkflowStatisticsDataLoaded } from '@/Interfaces'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { StatisticsRequest } from './workflow-statistics.types'; diff --git a/packages/cli/src/CrashJournal.ts b/packages/cli/src/crash-journal.ts similarity index 96% rename from packages/cli/src/CrashJournal.ts rename to packages/cli/src/crash-journal.ts index ab13f2cb12..dcba654a3d 100644 --- a/packages/cli/src/CrashJournal.ts +++ b/packages/cli/src/crash-journal.ts @@ -5,7 +5,7 @@ import { Container } from 'typedi'; import { InstanceSettings } from 'n8n-core'; import { sleep } from 'n8n-workflow'; import { inProduction } from '@/constants'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; export const touchFile = async (filePath: string): Promise => { await mkdir(dirname(filePath), { recursive: true }); diff --git a/packages/cli/src/CredentialTypes.ts b/packages/cli/src/credential-types.ts similarity index 96% rename from packages/cli/src/CredentialTypes.ts rename to packages/cli/src/credential-types.ts index d97825a672..8d665c7592 100644 --- a/packages/cli/src/CredentialTypes.ts +++ b/packages/cli/src/credential-types.ts @@ -7,7 +7,7 @@ import { type LoadedClass, } from 'n8n-workflow'; import { RESPONSE_ERROR_MESSAGES } from '@/constants'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; @Service() export class CredentialTypes implements ICredentialTypes { diff --git a/packages/cli/src/CredentialsHelper.ts b/packages/cli/src/credentials-helper.ts similarity index 99% rename from packages/cli/src/CredentialsHelper.ts rename to packages/cli/src/credentials-helper.ts index 43ccf970e0..aa0491955c 100644 --- a/packages/cli/src/CredentialsHelper.ts +++ b/packages/cli/src/credentials-helper.ts @@ -30,8 +30,8 @@ import { ICredentialsHelper, NodeHelpers, Workflow, ApplicationError } from 'n8n import type { ICredentialsDb } from '@/Interfaces'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; -import { CredentialTypes } from '@/CredentialTypes'; -import { CredentialsOverwrites } from '@/CredentialsOverwrites'; +import { CredentialTypes } from '@/credential-types'; +import { CredentialsOverwrites } from '@/credentials-overwrites'; import { RESPONSE_ERROR_MESSAGES } from './constants'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; diff --git a/packages/cli/src/CredentialsOverwrites.ts b/packages/cli/src/credentials-overwrites.ts similarity index 97% rename from packages/cli/src/CredentialsOverwrites.ts rename to packages/cli/src/credentials-overwrites.ts index b80a31d9ce..b8df1a848c 100644 --- a/packages/cli/src/CredentialsOverwrites.ts +++ b/packages/cli/src/credentials-overwrites.ts @@ -3,8 +3,8 @@ import { GlobalConfig } from '@n8n/config'; import type { ICredentialDataDecryptedObject } from 'n8n-workflow'; import { deepCopy, jsonParse } from 'n8n-workflow'; import type { ICredentialsOverwrite } from '@/Interfaces'; -import { CredentialTypes } from '@/CredentialTypes'; -import { Logger } from '@/Logger'; +import { CredentialTypes } from '@/credential-types'; +import { Logger } from '@/logger'; @Service() export class CredentialsOverwrites { diff --git a/packages/cli/src/credentials/__tests__/credentials.service.test.ts b/packages/cli/src/credentials/__tests__/credentials.service.test.ts index 18216df867..8a5ce77e15 100644 --- a/packages/cli/src/credentials/__tests__/credentials.service.test.ts +++ b/packages/cli/src/credentials/__tests__/credentials.service.test.ts @@ -2,8 +2,8 @@ import { CREDENTIAL_EMPTY_VALUE, type ICredentialType } from 'n8n-workflow'; import { mock } from 'jest-mock-extended'; import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; -import type { CredentialTypes } from '@/CredentialTypes'; -import { CredentialsService } from '../credentials.service'; +import type { CredentialTypes } from '@/credential-types'; +import { CredentialsService } from '@/credentials/credentials.service'; describe('CredentialsService', () => { const credType = mock({ diff --git a/packages/cli/src/credentials/credentials.controller.ts b/packages/cli/src/credentials/credentials.controller.ts index ec592519bb..b8dacf445f 100644 --- a/packages/cli/src/credentials/credentials.controller.ts +++ b/packages/cli/src/credentials/credentials.controller.ts @@ -5,12 +5,11 @@ import { In } from '@n8n/typeorm'; import { CredentialsService } from './credentials.service'; import { CredentialRequest } from '@/requests'; -import { InternalHooks } from '@/InternalHooks'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NamingService } from '@/services/naming.service'; -import { License } from '@/License'; +import { License } from '@/license'; import { EnterpriseCredentialsService } from './credentials.service.ee'; import { Delete, @@ -23,7 +22,7 @@ import { ProjectScope, } from '@/decorators'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { UserManagementMailer } from '@/UserManagement/email'; +import { UserManagementMailer } from '@/user-management/email'; import * as Db from '@/Db'; import * as utils from '@/utils'; import { listQueryMiddleware } from '@/middlewares'; @@ -31,7 +30,7 @@ import { SharedCredentialsRepository } from '@/databases/repositories/sharedCred import { SharedCredentials } from '@/databases/entities/SharedCredentials'; import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; import { z } from 'zod'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController('/credentials') export class CredentialsController { @@ -42,7 +41,6 @@ export class CredentialsController { private readonly namingService: NamingService, private readonly license: License, private readonly logger: Logger, - private readonly internalHooks: InternalHooks, private readonly userManagementMailer: UserManagementMailer, private readonly sharedCredentialsRepository: SharedCredentialsRepository, private readonly projectRelationRepository: ProjectRelationRepository, @@ -129,11 +127,11 @@ export class CredentialsController { const mergedCredentials = deepCopy(credentials); const decryptedData = this.credentialsService.decrypt(storedCredential); - // When a sharee opens a credential, the fields and the credential data are missing - // so the payload will be empty + // When a sharee (or project viewer) opens a credential, the fields and the + // credential data are missing so the payload will be empty // We need to replace the credential contents with the db version if that's the case // So the credential can be tested properly - this.credentialsService.replaceCredentialContentsForSharee( + await this.credentialsService.replaceCredentialContentsForSharee( req.user, storedCredential, decryptedData, @@ -162,18 +160,17 @@ export class CredentialsController { req.body.projectId, ); - void this.internalHooks.onUserCreatedCredentials({ - user: req.user, - credential_name: newCredential.name, - credential_type: credential.type, - credential_id: credential.id, - public_api: false, - }); + const project = await this.sharedCredentialsRepository.findCredentialOwningProject( + credential.id, + ); + this.eventService.emit('credentials-created', { user: req.user, - credentialName: newCredential.name, credentialType: credential.type, credentialId: credential.id, + publicApi: false, + projectId: project?.id, + projectType: project?.type, }); const scopes = await this.credentialsService.getCredentialScopes(req.user, credential.id); @@ -223,15 +220,8 @@ export class CredentialsController { this.logger.verbose('Credential updated', { credentialId }); - void this.internalHooks.onUserUpdatedCredentials({ - user: req.user, - credential_name: credential.name, - credential_type: credential.type, - credential_id: credential.id, - }); this.eventService.emit('credentials-updated', { user: req.user, - credentialName: credential.name, credentialType: credential.type, credentialId: credential.id, }); @@ -264,15 +254,8 @@ export class CredentialsController { await this.credentialsService.delete(credential); - void this.internalHooks.onUserDeletedCredentials({ - user: req.user, - credential_name: credential.name, - credential_type: credential.type, - credential_id: credential.id, - }); this.eventService.emit('credentials-deleted', { user: req.user, - credentialName: credential.name, credentialType: credential.type, credentialId: credential.id, }); @@ -308,25 +291,22 @@ export class CredentialsController { let newShareeIds: string[] = []; await Db.transaction(async (trx) => { - const currentPersonalProjectIDs = credential.shared + const currentProjectIds = credential.shared .filter((sc) => sc.role === 'credential:user') .map((sc) => sc.projectId); - const newPersonalProjectIds = shareWithIds; + const newProjectIds = shareWithIds; - const toShare = utils.rightDiff( - [currentPersonalProjectIDs, (id) => id], - [newPersonalProjectIds, (id) => id], - ); + const toShare = utils.rightDiff([currentProjectIds, (id) => id], [newProjectIds, (id) => id]); const toUnshare = utils.rightDiff( - [newPersonalProjectIds, (id) => id], - [currentPersonalProjectIDs, (id) => id], + [newProjectIds, (id) => id], + [currentProjectIds, (id) => id], ); const deleteResult = await trx.delete(SharedCredentials, { credentialsId: credentialId, projectId: In(toUnshare), }); - await this.enterpriseCredentialsService.shareWithProjects(credential, toShare, trx); + await this.enterpriseCredentialsService.shareWithProjects(req.user, credential, toShare, trx); if (deleteResult.affected) { amountRemoved = deleteResult.affected; @@ -335,22 +315,12 @@ export class CredentialsController { newShareeIds = toShare; }); - void this.internalHooks.onUserSharedCredentials({ - user: req.user, - credential_name: credential.name, - credential_type: credential.type, - credential_id: credential.id, - user_id_sharer: req.user.id, - user_ids_sharees_added: newShareeIds, - sharees_removed: amountRemoved, - }); this.eventService.emit('credentials-shared', { user: req.user, - credentialName: credential.name, credentialType: credential.type, credentialId: credential.id, userIdSharer: req.user.id, - userIdsShareesRemoved: newShareeIds, + userIdsShareesAdded: newShareeIds, shareesRemoved: amountRemoved, }); diff --git a/packages/cli/src/credentials/credentials.service.ee.ts b/packages/cli/src/credentials/credentials.service.ee.ts index 981ecc5d59..44fc614cbd 100644 --- a/packages/cli/src/credentials/credentials.service.ee.ts +++ b/packages/cli/src/credentials/credentials.service.ee.ts @@ -12,6 +12,7 @@ import { Project } from '@/databases/entities/Project'; import { ProjectService } from '@/services/project.service'; import { TransferCredentialError } from '@/errors/response-errors/transfer-credential.error'; import { SharedCredentials } from '@/databases/entities/SharedCredentials'; +import { RoleService } from '@/services/role.service'; @Service() export class EnterpriseCredentialsService { @@ -20,9 +21,11 @@ export class EnterpriseCredentialsService { private readonly ownershipService: OwnershipService, private readonly credentialsService: CredentialsService, private readonly projectService: ProjectService, + private readonly roleService: RoleService, ) {} async shareWithProjects( + user: User, credential: CredentialsEntity, shareWithIds: string[], entityManager?: EntityManager, @@ -30,19 +33,35 @@ export class EnterpriseCredentialsService { const em = entityManager ?? this.sharedCredentialsRepository.manager; const projects = await em.find(Project, { - where: { id: In(shareWithIds), type: 'personal' }, + where: [ + { + id: In(shareWithIds), + type: 'team', + // if user can see all projects, don't check project access + // if they can't, find projects they can list + ...(user.hasGlobalScope('project:list') + ? {} + : { + projectRelations: { + userId: user.id, + role: In(this.roleService.rolesWithScope('project', 'project:list')), + }, + }), + }, + { + id: In(shareWithIds), + type: 'personal', + }, + ], }); - const newSharedCredentials = projects - // We filter by role === 'project:personalOwner' above and there should - // always only be one owner. - .map((project) => - this.sharedCredentialsRepository.create({ - credentialsId: credential.id, - role: 'credential:user', - projectId: project.id, - }), - ); + const newSharedCredentials = projects.map((project) => + this.sharedCredentialsRepository.create({ + credentialsId: credential.id, + role: 'credential:user', + projectId: project.id, + }), + ); return await em.save(newSharedCredentials); } diff --git a/packages/cli/src/credentials/credentials.service.ts b/packages/cli/src/credentials/credentials.service.ts index 0ecfd31877..e5d1a53252 100644 --- a/packages/cli/src/credentials/credentials.service.ts +++ b/packages/cli/src/credentials/credentials.service.ts @@ -16,17 +16,17 @@ import { import type { Scope } from '@n8n/permissions'; import * as Db from '@/Db'; import type { ICredentialsDb } from '@/Interfaces'; -import { createCredentialsFromCredentialsEntity } from '@/CredentialsHelper'; +import { createCredentialsFromCredentialsEntity } from '@/credentials-helper'; import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; import { CredentialsEntity } from '@db/entities/CredentialsEntity'; import { SharedCredentials } from '@db/entities/SharedCredentials'; -import { validateEntity } from '@/GenericHelpers'; -import { ExternalHooks } from '@/ExternalHooks'; +import { validateEntity } from '@/generic-helpers'; +import { ExternalHooks } from '@/external-hooks'; import type { User } from '@db/entities/User'; import type { CredentialRequest, ListQuery } from '@/requests'; -import { CredentialTypes } from '@/CredentialTypes'; +import { CredentialTypes } from '@/credential-types'; import { OwnershipService } from '@/services/ownership.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { Service } from 'typedi'; @@ -38,6 +38,7 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error'; import type { ProjectRelation } from '@/databases/entities/ProjectRelation'; import { RoleService } from '@/services/role.service'; import { UserRepository } from '@/databases/repositories/user.repository'; +import { userHasScope } from '@/permissions/check-access'; export type CredentialsGetSharedOptions = | { allowGlobalScope: true; globalScope: Scope } @@ -90,6 +91,19 @@ export class CredentialsService { let credentials = await this.credentialsRepository.findMany(options.listQueryOptions); if (isDefaultSelect) { + // Since we're filtering using project ID as part of the relation, + // we end up filtering out all the other relations, meaning that if + // it's shared to a project, it won't be able to find the home project. + // To solve this, we have to get all the relation now, even though + // we're deleting them later. + if ((options.listQueryOptions?.filter?.shared as { projectId?: string })?.projectId) { + const relations = await this.sharedCredentialsRepository.getAllRelationsForCredentials( + credentials.map((c) => c.id), + ); + credentials.forEach((c) => { + c.shared = relations.filter((r) => r.credentialsId === c.id); + }); + } credentials = credentials.map((c) => this.ownershipService.addOwnedByAndSharedWith(c)); } @@ -130,6 +144,20 @@ export class CredentialsService { ); if (isDefaultSelect) { + // Since we're filtering using project ID as part of the relation, + // we end up filtering out all the other relations, meaning that if + // it's shared to a project, it won't be able to find the home project. + // To solve this, we have to get all the relation now, even though + // we're deleting them later. + if ((options.listQueryOptions?.filter?.shared as { projectId?: string })?.projectId) { + const relations = await this.sharedCredentialsRepository.getAllRelationsForCredentials( + credentials.map((c) => c.id), + ); + credentials.forEach((c) => { + c.shared = relations.filter((r) => r.credentialsId === c.id); + }); + } + credentials = credentials.map((c) => this.ownershipService.addOwnedByAndSharedWith(c)); } @@ -572,28 +600,20 @@ export class CredentialsService { ); } - replaceCredentialContentsForSharee( + async replaceCredentialContentsForSharee( user: User, credential: CredentialsEntity, decryptedData: ICredentialDataDecryptedObject, mergedCredentials: ICredentialsDecrypted, ) { - credential.shared.forEach((sharedCredentials) => { - if (sharedCredentials.role === 'credential:owner') { - if (sharedCredentials.project.type === 'personal') { - // Find the owner of this personal project - sharedCredentials.project.projectRelations.forEach((projectRelation) => { - if ( - projectRelation.role === 'project:personalOwner' && - projectRelation.user.id !== user.id - ) { - // If we realize that the current user does not own this credential - // We replace the payload with the stored decrypted data - mergedCredentials.data = decryptedData; - } - }); - } - } - }); + // We may want to change this to 'credential:decrypt' if that gets added, but this + // works for now. The only time we wouldn't want to do this is if the user + // could actually be testing the credential before saving it, so this should cover + // the cases we need it for. + if ( + !(await userHasScope(user, ['credential:update'], false, { credentialId: credential.id })) + ) { + mergedCredentials.data = decryptedData; + } } } diff --git a/packages/cli/src/databases/entities/InvalidAuthToken.ts b/packages/cli/src/databases/entities/InvalidAuthToken.ts new file mode 100644 index 0000000000..e21860d146 --- /dev/null +++ b/packages/cli/src/databases/entities/InvalidAuthToken.ts @@ -0,0 +1,11 @@ +import { Column, Entity, PrimaryColumn } from '@n8n/typeorm'; +import { datetimeColumnType } from './AbstractEntity'; + +@Entity() +export class InvalidAuthToken { + @PrimaryColumn() + token: string; + + @Column(datetimeColumnType) + expiresAt: Date; +} diff --git a/packages/cli/src/databases/entities/User.ts b/packages/cli/src/databases/entities/User.ts index 5755f24d84..dad8bbbe80 100644 --- a/packages/cli/src/databases/entities/User.ts +++ b/packages/cli/src/databases/entities/User.ts @@ -13,7 +13,7 @@ import { IsEmail, IsString, Length } from 'class-validator'; import type { IUser, IUserSettings } from 'n8n-workflow'; import type { SharedWorkflow } from './SharedWorkflow'; import type { SharedCredentials } from './SharedCredentials'; -import { NoXss } from '../utils/customValidators'; +import { NoXss } from '@/validators/no-xss.validator'; import { objectRetriever, lowerCaser } from '../utils/transformers'; import { WithTimestamps, jsonColumnType } from './AbstractEntity'; import type { IPersonalizationSurveyAnswers } from '@/Interfaces'; @@ -25,6 +25,7 @@ import { } from '@/permissions/global-roles'; import { hasScope, type ScopeOptions, type Scope } from '@n8n/permissions'; import type { ProjectRelation } from './ProjectRelation'; +import { NoUrl } from '@/validators/no-url.validator'; export type GlobalRole = 'global:owner' | 'global:admin' | 'global:member'; export type AssignableRole = Exclude; @@ -51,12 +52,14 @@ export class User extends WithTimestamps implements IUser { @Column({ length: 32, nullable: true }) @NoXss() + @NoUrl() @IsString({ message: 'First name must be of type string.' }) @Length(1, 32, { message: 'First name must be $constraint1 to $constraint2 characters long.' }) firstName: string; @Column({ length: 32, nullable: true }) @NoXss() + @NoUrl() @IsString({ message: 'Last name must be of type string.' }) @Length(1, 32, { message: 'Last name must be $constraint1 to $constraint2 characters long.' }) lastName: string; @@ -104,7 +107,7 @@ export class User extends WithTimestamps implements IUser { @Column({ type: String, nullable: true }) @Index({ unique: true }) - apiKey?: string | null; + apiKey: string | null; @Column({ type: Boolean, default: false }) mfaEnabled: boolean; diff --git a/packages/cli/test/unit/databases/entities/user.entity.test.ts b/packages/cli/src/databases/entities/__tests__/user.entity.test.ts similarity index 100% rename from packages/cli/test/unit/databases/entities/user.entity.test.ts rename to packages/cli/src/databases/entities/__tests__/user.entity.test.ts diff --git a/packages/cli/src/databases/entities/index.ts b/packages/cli/src/databases/entities/index.ts index db8b113baf..bd7d187486 100644 --- a/packages/cli/src/databases/entities/index.ts +++ b/packages/cli/src/databases/entities/index.ts @@ -21,6 +21,7 @@ import { ExecutionData } from './ExecutionData'; import { WorkflowHistory } from './WorkflowHistory'; import { Project } from './Project'; import { ProjectRelation } from './ProjectRelation'; +import { InvalidAuthToken } from './InvalidAuthToken'; export const entities = { AuthIdentity, @@ -31,6 +32,7 @@ export const entities = { ExecutionEntity, InstalledNodes, InstalledPackages, + InvalidAuthToken, Settings, SharedCredentials, SharedWorkflow, diff --git a/packages/cli/src/databases/migrations/common/1674509946020-CreateLdapEntities.ts b/packages/cli/src/databases/migrations/common/1674509946020-CreateLdapEntities.ts index 2115cbeffd..1d915a5576 100644 --- a/packages/cli/src/databases/migrations/common/1674509946020-CreateLdapEntities.ts +++ b/packages/cli/src/databases/migrations/common/1674509946020-CreateLdapEntities.ts @@ -1,5 +1,5 @@ import type { MigrationContext, ReversibleMigration } from '@db/types'; -import { LDAP_DEFAULT_CONFIGURATION, LDAP_FEATURE_NAME } from '@/Ldap/constants'; +import { LDAP_DEFAULT_CONFIGURATION, LDAP_FEATURE_NAME } from '@/ldap/constants'; export class CreateLdapEntities1674509946020 implements ReversibleMigration { async up({ escape, dbType, isMysql, runQuery }: MigrationContext) { diff --git a/packages/cli/src/databases/migrations/common/1723627610222-CreateInvalidAuthTokenTable.ts b/packages/cli/src/databases/migrations/common/1723627610222-CreateInvalidAuthTokenTable.ts new file mode 100644 index 0000000000..f28696c199 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1723627610222-CreateInvalidAuthTokenTable.ts @@ -0,0 +1,16 @@ +import type { MigrationContext, ReversibleMigration } from '@db/types'; + +const tableName = 'invalid_auth_token'; + +export class CreateInvalidAuthTokenTable1723627610222 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column } }: MigrationContext) { + await createTable(tableName).withColumns( + column('token').varchar(512).primary, + column('expiresAt').timestamp().notNull, + ); + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable(tableName); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index ecd5f66a7c..b4900eb59d 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -59,6 +59,7 @@ import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNo import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; +import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -121,4 +122,5 @@ export const mysqlMigrations: Migration[] = [ MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, + CreateInvalidAuthTokenTable1723627610222, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 720c79a8e3..85bd58f371 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -59,6 +59,7 @@ import { MakeExecutionStatusNonNullable1714133768521 } from '../common/171413376 import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; import { FixExecutionMetadataSequence1721377157740 } from './1721377157740-FixExecutionMetadataSequence'; +import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -121,4 +122,5 @@ export const postgresMigrations: Migration[] = [ AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, FixExecutionMetadataSequence1721377157740, + CreateInvalidAuthTokenTable1723627610222, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 15000a78e0..974c743d0f 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -56,6 +56,7 @@ import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNo import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; +import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -115,6 +116,7 @@ const sqliteMigrations: Migration[] = [ MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, + CreateInvalidAuthTokenTable1723627610222, ]; export { sqliteMigrations }; diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts similarity index 95% rename from packages/cli/test/unit/repositories/execution.repository.test.ts rename to packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts index f8aba2e8a1..1a4929f5cd 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts @@ -8,8 +8,7 @@ import { mock } from 'jest-mock-extended'; import { ExecutionEntity } from '@db/entities/ExecutionEntity'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { mockEntityManager } from '../../shared/mocking'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance, mockEntityManager } from '@test/mocking'; describe('ExecutionRepository', () => { const entityManager = mockEntityManager(ExecutionEntity); diff --git a/packages/cli/test/unit/repositories/sharedCredentials.repository.test.ts b/packages/cli/src/databases/repositories/__tests__/sharedCredentials.repository.test.ts similarity index 98% rename from packages/cli/test/unit/repositories/sharedCredentials.repository.test.ts rename to packages/cli/src/databases/repositories/__tests__/sharedCredentials.repository.test.ts index 8eb8b49814..6fb4bad4ea 100644 --- a/packages/cli/test/unit/repositories/sharedCredentials.repository.test.ts +++ b/packages/cli/src/databases/repositories/__tests__/sharedCredentials.repository.test.ts @@ -8,7 +8,7 @@ import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; import { SharedCredentials } from '@db/entities/SharedCredentials'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { GLOBAL_MEMBER_SCOPES, GLOBAL_OWNER_SCOPES } from '@/permissions/global-roles'; -import { mockEntityManager } from '../../shared/mocking'; +import { mockEntityManager } from '@test/mocking'; describe('SharedCredentialsRepository', () => { const entityManager = mockEntityManager(SharedCredentials); diff --git a/packages/cli/test/unit/repositories/workflowStatistics.test.ts b/packages/cli/src/databases/repositories/__tests__/workflowStatistics.test.ts similarity index 96% rename from packages/cli/test/unit/repositories/workflowStatistics.test.ts rename to packages/cli/src/databases/repositories/__tests__/workflowStatistics.test.ts index 86e0ee1c92..7bed056549 100644 --- a/packages/cli/test/unit/repositories/workflowStatistics.test.ts +++ b/packages/cli/src/databases/repositories/__tests__/workflowStatistics.test.ts @@ -5,7 +5,7 @@ import { mock, mockClear } from 'jest-mock-extended'; import { StatisticsNames, WorkflowStatistics } from '@db/entities/WorkflowStatistics'; import { WorkflowStatisticsRepository } from '@db/repositories/workflowStatistics.repository'; -import { mockEntityManager } from '../../shared/mocking'; +import { mockEntityManager } from '@test/mocking'; describe('insertWorkflowStatistics', () => { const entityManager = mockEntityManager(WorkflowStatistics); diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 1ebb22d8eb..364619fab6 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -42,7 +42,7 @@ import type { ExecutionData } from '../entities/ExecutionData'; import { ExecutionEntity } from '../entities/ExecutionEntity'; import { ExecutionMetadata } from '../entities/ExecutionMetadata'; import { ExecutionDataRepository } from './executionData.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { ExecutionSummaries } from '@/executions/execution.types'; import { PostgresLiveRowsRetrievalError } from '@/errors/postgres-live-rows-retrieval.error'; import { separate } from '@/utils'; @@ -270,6 +270,9 @@ export class ExecutionRepository extends Repository { return rest; } + /** + * Insert a new execution and its execution data using a transaction. + */ async createNewExecution(execution: ExecutionPayload): Promise { const { data, workflowData, ...rest } = execution; const { identifiers: inserted } = await this.insert(rest); diff --git a/packages/cli/src/databases/repositories/invalidAuthToken.repository.ts b/packages/cli/src/databases/repositories/invalidAuthToken.repository.ts new file mode 100644 index 0000000000..c6340ba88a --- /dev/null +++ b/packages/cli/src/databases/repositories/invalidAuthToken.repository.ts @@ -0,0 +1,10 @@ +import { Service } from 'typedi'; +import { DataSource, Repository } from '@n8n/typeorm'; +import { InvalidAuthToken } from '../entities/InvalidAuthToken'; + +@Service() +export class InvalidAuthTokenRepository extends Repository { + constructor(dataSource: DataSource) { + super(InvalidAuthToken, dataSource.manager); + } +} diff --git a/packages/cli/src/databases/repositories/projectRelation.repository.ts b/packages/cli/src/databases/repositories/projectRelation.repository.ts index bddfd6e38d..1f875d011f 100644 --- a/packages/cli/src/databases/repositories/projectRelation.repository.ts +++ b/packages/cli/src/databases/repositories/projectRelation.repository.ts @@ -52,4 +52,21 @@ export class ProjectRelationRepository extends Repository { {} as Record, ); } + + async findUserIdsByProjectId(projectId: string): Promise { + const rows = await this.find({ + select: ['userId'], + where: { projectId }, + }); + + return [...new Set(rows.map((r) => r.userId))]; + } + + async findAllByUser(userId: string) { + return await this.find({ + where: { + userId, + }, + }); + } } diff --git a/packages/cli/src/databases/repositories/settings.repository.ts b/packages/cli/src/databases/repositories/settings.repository.ts index 94cb32305a..6a745a8a70 100644 --- a/packages/cli/src/databases/repositories/settings.repository.ts +++ b/packages/cli/src/databases/repositories/settings.repository.ts @@ -1,4 +1,4 @@ -import { EXTERNAL_SECRETS_DB_KEY } from '@/ExternalSecrets/constants'; +import { EXTERNAL_SECRETS_DB_KEY } from '@/external-secrets/constants'; import { Service } from 'typedi'; import { DataSource, Repository } from '@n8n/typeorm'; import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; diff --git a/packages/cli/src/databases/repositories/sharedCredentials.repository.ts b/packages/cli/src/databases/repositories/sharedCredentials.repository.ts index 8d2d1fa7af..03acc24823 100644 --- a/packages/cli/src/databases/repositories/sharedCredentials.repository.ts +++ b/packages/cli/src/databases/repositories/sharedCredentials.repository.ts @@ -151,4 +151,13 @@ export class SharedCredentialsRepository extends Repository { }) )?.project; } + + async getAllRelationsForCredentials(credentialIds: string[]) { + return await this.find({ + where: { + credentialsId: In(credentialIds), + }, + relations: ['project'], + }); + } } diff --git a/packages/cli/src/databases/repositories/sharedWorkflow.repository.ts b/packages/cli/src/databases/repositories/sharedWorkflow.repository.ts index f8ff3523b2..d8e224fee2 100644 --- a/packages/cli/src/databases/repositories/sharedWorkflow.repository.ts +++ b/packages/cli/src/databases/repositories/sharedWorkflow.repository.ts @@ -175,7 +175,7 @@ export class SharedWorkflowRepository extends Repository { }, }); - return sharedWorkflows.map((sw) => sw.workflow); + return sharedWorkflows.map((sw) => ({ ...sw.workflow, projectId: sw.projectId })); } /** @@ -200,4 +200,13 @@ export class SharedWorkflowRepository extends Repository { }) )?.project; } + + async getRelationsByWorkflowIdsAndProjectIds(workflowIds: string[], projectIds: string[]) { + return await this.find({ + where: { + workflowId: In(workflowIds), + projectId: In(projectIds), + }, + }); + } } diff --git a/packages/cli/src/databases/subscribers/UserSubscriber.ts b/packages/cli/src/databases/subscribers/UserSubscriber.ts index b925965a0c..52ef95871e 100644 --- a/packages/cli/src/databases/subscribers/UserSubscriber.ts +++ b/packages/cli/src/databases/subscribers/UserSubscriber.ts @@ -2,7 +2,7 @@ import { Container } from 'typedi'; import type { EntitySubscriberInterface, UpdateEvent } from '@n8n/typeorm'; import { EventSubscriber } from '@n8n/typeorm'; import { ApplicationError, ErrorReporterProxy } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { Project } from '../entities/Project'; import { User } from '../entities/User'; diff --git a/packages/cli/src/databases/types.ts b/packages/cli/src/databases/types.ts index e1dc8f20e9..f4c3db0452 100644 --- a/packages/cli/src/databases/types.ts +++ b/packages/cli/src/databases/types.ts @@ -1,6 +1,6 @@ import type { INodeTypes } from 'n8n-workflow'; import type { QueryRunner, ObjectLiteral } from '@n8n/typeorm'; -import type { Logger } from '@/Logger'; +import type { Logger } from '@/logger'; import type { createSchemaBuilder } from './dsl'; export type DatabaseType = 'mariadb' | 'postgresdb' | 'mysqldb' | 'sqlite'; diff --git a/packages/cli/test/unit/databases/utils/migrationHelpers.test.ts b/packages/cli/src/databases/utils/__tests__/migrationHelpers.test.ts similarity index 100% rename from packages/cli/test/unit/databases/utils/migrationHelpers.test.ts rename to packages/cli/src/databases/utils/__tests__/migrationHelpers.test.ts diff --git a/packages/cli/src/databases/utils/customValidators.ts b/packages/cli/src/databases/utils/customValidators.ts deleted file mode 100644 index e98c507e92..0000000000 --- a/packages/cli/src/databases/utils/customValidators.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { registerDecorator } from 'class-validator'; - -export function NoXss() { - return (object: object, propertyName: string): void => { - registerDecorator({ - name: 'NoXss', - target: object.constructor, - propertyName, - constraints: [propertyName], - options: { message: `Malicious ${propertyName}` }, - validator: { - validate(value: string) { - return !/(^http|^www)|<(\s*)?(script|a)|(\.[\p{L}\d-]+)/u.test(value); - }, - }, - }); - }; -} diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts index b3187c44e5..0f116590ee 100644 --- a/packages/cli/src/databases/utils/migrationHelpers.ts +++ b/packages/cli/src/databases/utils/migrationHelpers.ts @@ -9,8 +9,8 @@ import { ApplicationError, jsonParse } from 'n8n-workflow'; import { inTest } from '@/constants'; import type { BaseMigration, Migration, MigrationContext, MigrationFn } from '@db/types'; import { createSchemaBuilder } from '@db/dsl'; -import { NodeTypes } from '@/NodeTypes'; -import { Logger } from '@/Logger'; +import { NodeTypes } from '@/node-types'; +import { Logger } from '@/logger'; const PERSONALIZATION_SURVEY_FILENAME = 'personalizationSurvey.json'; diff --git a/packages/cli/test/unit/decorators/controller.registry.test.ts b/packages/cli/src/decorators/__tests__/controller.registry.test.ts similarity index 92% rename from packages/cli/test/unit/decorators/controller.registry.test.ts rename to packages/cli/src/decorators/__tests__/controller.registry.test.ts index 04b4884dcc..3a0f0b91d2 100644 --- a/packages/cli/test/unit/decorators/controller.registry.test.ts +++ b/packages/cli/src/decorators/__tests__/controller.registry.test.ts @@ -8,18 +8,20 @@ import { mock } from 'jest-mock-extended'; import { ControllerRegistry, Get, Licensed, RestController } from '@/decorators'; import type { AuthService } from '@/auth/auth.service'; -import type { License } from '@/License'; +import type { License } from '@/license'; import type { SuperAgentTest } from '@test-integration/types'; +import type { GlobalConfig } from '@n8n/config'; describe('ControllerRegistry', () => { const license = mock(); const authService = mock(); + const globalConfig = mock({ endpoints: { rest: 'rest' } }); let agent: SuperAgentTest; beforeEach(() => { jest.resetAllMocks(); const app = express(); - new ControllerRegistry(license, authService).activate(app); + new ControllerRegistry(license, authService, globalConfig).activate(app); agent = testAgent(app); }); diff --git a/packages/cli/test/unit/decorators/OnShutdown.test.ts b/packages/cli/src/decorators/__tests__/on-shutdown.test.ts similarity index 95% rename from packages/cli/test/unit/decorators/OnShutdown.test.ts rename to packages/cli/src/decorators/__tests__/on-shutdown.test.ts index 833915f4cb..f8d11140cf 100644 --- a/packages/cli/test/unit/decorators/OnShutdown.test.ts +++ b/packages/cli/src/decorators/__tests__/on-shutdown.test.ts @@ -1,6 +1,6 @@ import Container, { Service } from 'typedi'; -import { OnShutdown } from '@/decorators/OnShutdown'; -import { ShutdownService } from '@/shutdown/Shutdown.service'; +import { OnShutdown } from '@/decorators/on-shutdown'; +import { ShutdownService } from '@/shutdown/shutdown.service'; import { mock } from 'jest-mock-extended'; describe('OnShutdown', () => { diff --git a/packages/cli/src/decorators/controller.registry.ts b/packages/cli/src/decorators/controller.registry.ts index c012922c16..6ab4e9191d 100644 --- a/packages/cli/src/decorators/controller.registry.ts +++ b/packages/cli/src/decorators/controller.registry.ts @@ -4,15 +4,14 @@ import type { Application, Request, Response, RequestHandler } from 'express'; import { rateLimit as expressRateLimit } from 'express-rate-limit'; import { AuthService } from '@/auth/auth.service'; -import config from '@/config'; import { UnauthenticatedError } from '@/errors/response-errors/unauthenticated.error'; import { inProduction, RESPONSE_ERROR_MESSAGES } from '@/constants'; import type { BooleanLicenseFeature } from '@/Interfaces'; -import { License } from '@/License'; +import { License } from '@/license'; import type { AuthenticatedRequest } from '@/requests'; -import { send } from '@/ResponseHelper'; // TODO: move `ResponseHelper.send` to this file -import { userHasScope } from '@/permissions/checkAccess'; - +import { send } from '@/response-helper'; // TODO: move `ResponseHelper.send` to this file +import { userHasScope } from '@/permissions/check-access'; +import { GlobalConfig } from '@n8n/config'; import type { AccessScope, Controller, @@ -52,6 +51,7 @@ export class ControllerRegistry { constructor( private readonly license: License, private readonly authService: AuthService, + private readonly globalConfig: GlobalConfig, ) {} activate(app: Application) { @@ -64,7 +64,7 @@ export class ControllerRegistry { const metadata = registry.get(controllerClass)!; const router = Router({ mergeParams: true }); - const prefix = `/${config.getEnv('endpoints.rest')}/${metadata.basePath}` + const prefix = `/${this.globalConfig.endpoints.rest}/${metadata.basePath}` .replace(/\/+/g, '/') .replace(/\/$/, ''); app.use(prefix, router); diff --git a/packages/cli/src/decorators/index.ts b/packages/cli/src/decorators/index.ts index 61edd6d9d9..5ef6c15f52 100644 --- a/packages/cli/src/decorators/index.ts +++ b/packages/cli/src/decorators/index.ts @@ -1,6 +1,6 @@ -export { RestController } from './RestController'; -export { Get, Post, Put, Patch, Delete } from './Route'; -export { Middleware } from './Middleware'; +export { RestController } from './rest-controller'; +export { Get, Post, Put, Patch, Delete } from './route'; +export { Middleware } from './middleware'; export { ControllerRegistry } from './controller.registry'; -export { Licensed } from './Licensed'; -export { GlobalScope, ProjectScope } from './Scoped'; +export { Licensed } from './licensed'; +export { GlobalScope, ProjectScope } from './scoped'; diff --git a/packages/cli/src/decorators/Licensed.ts b/packages/cli/src/decorators/licensed.ts similarity index 100% rename from packages/cli/src/decorators/Licensed.ts rename to packages/cli/src/decorators/licensed.ts diff --git a/packages/cli/src/decorators/Middleware.ts b/packages/cli/src/decorators/middleware.ts similarity index 100% rename from packages/cli/src/decorators/Middleware.ts rename to packages/cli/src/decorators/middleware.ts diff --git a/packages/cli/src/decorators/OnShutdown.ts b/packages/cli/src/decorators/on-shutdown.ts similarity index 95% rename from packages/cli/src/decorators/OnShutdown.ts rename to packages/cli/src/decorators/on-shutdown.ts index 68b3cbb4ca..ec2d955fe6 100644 --- a/packages/cli/src/decorators/OnShutdown.ts +++ b/packages/cli/src/decorators/on-shutdown.ts @@ -1,6 +1,6 @@ import { Container } from 'typedi'; import { ApplicationError } from 'n8n-workflow'; -import { type ServiceClass, ShutdownService } from '@/shutdown/Shutdown.service'; +import { type ServiceClass, ShutdownService } from '@/shutdown/shutdown.service'; import { DEFAULT_SHUTDOWN_PRIORITY } from '@/constants'; /** diff --git a/packages/cli/src/decorators/Redactable.ts b/packages/cli/src/decorators/redactable.ts similarity index 94% rename from packages/cli/src/decorators/Redactable.ts rename to packages/cli/src/decorators/redactable.ts index e5debeb7a1..51d02c5c3d 100644 --- a/packages/cli/src/decorators/Redactable.ts +++ b/packages/cli/src/decorators/redactable.ts @@ -1,5 +1,5 @@ import { RedactableError } from '@/errors/redactable.error'; -import type { UserLike } from '@/eventbus/event.types'; +import type { UserLike } from '@/events/relay-event-map'; function toRedactable(userLike: UserLike) { return { @@ -14,7 +14,7 @@ function toRedactable(userLike: UserLike) { type FieldName = 'user' | 'inviter' | 'invitee'; /** - * Mark redactable properties in a `{ user: UserLike }` field in an `AuditEventRelay` + * Mark redactable properties in a `{ user: UserLike }` field in an `LogStreamingEventRelay` * method arg. These properties will be later redacted by the log streaming * destination based on user prefs. Only for `n8n.audit.*` logs. * diff --git a/packages/cli/src/decorators/RestController.ts b/packages/cli/src/decorators/rest-controller.ts similarity index 100% rename from packages/cli/src/decorators/RestController.ts rename to packages/cli/src/decorators/rest-controller.ts diff --git a/packages/cli/src/decorators/Route.ts b/packages/cli/src/decorators/route.ts similarity index 100% rename from packages/cli/src/decorators/Route.ts rename to packages/cli/src/decorators/route.ts diff --git a/packages/cli/src/decorators/Scoped.ts b/packages/cli/src/decorators/scoped.ts similarity index 100% rename from packages/cli/src/decorators/Scoped.ts rename to packages/cli/src/decorators/scoped.ts diff --git a/packages/cli/src/environments/sourceControl/__tests__/source-control-export.service.test.ts b/packages/cli/src/environments/sourceControl/__tests__/source-control-export.service.test.ts new file mode 100644 index 0000000000..7746875f63 --- /dev/null +++ b/packages/cli/src/environments/sourceControl/__tests__/source-control-export.service.test.ts @@ -0,0 +1,85 @@ +import mock from 'jest-mock-extended/lib/Mock'; +import { SourceControlExportService } from '../sourceControlExport.service.ee'; +import type { SourceControlledFile } from '../types/sourceControlledFile'; +import { Cipher, type InstanceSettings } from 'n8n-core'; +import { SharedCredentialsRepository } from '@/databases/repositories/sharedCredentials.repository'; +import { mockInstance } from '@test/mocking'; + +import type { SharedCredentials } from '@/databases/entities/SharedCredentials'; +import type { CredentialsEntity } from '@/databases/entities/CredentialsEntity'; +import Container from 'typedi'; +import { ApplicationError, deepCopy } from 'n8n-workflow'; + +// https://github.com/jestjs/jest/issues/4715 +function deepSpyOn(object: O, methodName: M) { + const spy = jest.fn(); + // eslint-disable-next-line @typescript-eslint/ban-types + const originalMethod = object[methodName]; + + if (typeof originalMethod !== 'function') { + throw new ApplicationError(`${methodName.toString()} is not a function`, { level: 'warning' }); + } + + object[methodName] = function (...args: unknown[]) { + const clonedArgs = deepCopy(args); + spy(...clonedArgs); + return originalMethod.apply(this, args); + } as O[M]; + + return spy; +} + +describe('SourceControlExportService', () => { + const service = new SourceControlExportService( + mock(), + mock(), + mock(), + mock({ n8nFolder: '' }), + ); + + describe('exportCredentialsToWorkFolder', () => { + it('should export credentials to work folder', async () => { + /** + * Arrange + */ + // @ts-expect-error Private method + const replaceSpy = deepSpyOn(service, 'replaceCredentialData'); + + mockInstance(SharedCredentialsRepository).findByCredentialIds.mockResolvedValue([ + mock({ + credentials: mock({ + data: Container.get(Cipher).encrypt( + JSON.stringify({ + authUrl: 'test', + accessTokenUrl: 'test', + clientId: 'test', + clientSecret: 'test', + oauthTokenData: { + access_token: 'test', + token_type: 'test', + expires_in: 123, + refresh_token: 'test', + }, + }), + ), + }), + }), + ]); + + /** + * Act + */ + await service.exportCredentialsToWorkFolder([mock()]); + + /** + * Assert + */ + expect(replaceSpy).toHaveBeenCalledWith({ + authUrl: 'test', + accessTokenUrl: 'test', + clientId: 'test', + clientSecret: 'test', + }); + }); + }); +}); diff --git a/packages/cli/test/unit/GitService.test.ts b/packages/cli/src/environments/sourceControl/__tests__/sourceControlGit.service.test.ts similarity index 100% rename from packages/cli/test/unit/GitService.test.ts rename to packages/cli/src/environments/sourceControl/__tests__/sourceControlGit.service.test.ts diff --git a/packages/cli/test/unit/SourceControl.test.ts b/packages/cli/src/environments/sourceControl/__tests__/sourceControlHelper.ee.test.ts similarity index 98% rename from packages/cli/test/unit/SourceControl.test.ts rename to packages/cli/src/environments/sourceControl/__tests__/sourceControlHelper.ee.test.ts index 5eeccbb1b9..8b81bec7b2 100644 --- a/packages/cli/test/unit/SourceControl.test.ts +++ b/packages/cli/src/environments/sourceControl/__tests__/sourceControlHelper.ee.test.ts @@ -7,7 +7,7 @@ import { getTrackingInformationFromPullResult, sourceControlFoldersExistCheck, } from '@/environments/sourceControl/sourceControlHelper.ee'; -import { License } from '@/License'; +import { License } from '@/license'; import { SourceControlPreferencesService } from '@/environments/sourceControl/sourceControlPreferences.service.ee'; import { InstanceSettings } from 'n8n-core'; import path from 'path'; @@ -18,7 +18,7 @@ import { import { constants as fsConstants, accessSync } from 'fs'; import type { SourceControlledFile } from '@/environments/sourceControl/types/sourceControlledFile'; import type { SourceControlPreferences } from '@/environments/sourceControl/types/sourceControlPreferences'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; const pushResult: SourceControlledFile[] = [ { diff --git a/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts b/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts index 0a1db892f6..f92b0bfb1f 100644 --- a/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts @@ -12,7 +12,7 @@ import type { SourceControlPreferences } from './types/sourceControlPreferences' import type { SourceControlledFile } from './types/sourceControlledFile'; import { SOURCE_CONTROL_DEFAULT_BRANCH } from './constants'; import type { ImportResult } from './types/importResult'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; import { getRepoType } from './sourceControlHelper.ee'; import { SourceControlGetStatus } from './types/sourceControlGetStatus'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; diff --git a/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts index ac226a1b2e..375352e4cc 100644 --- a/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts @@ -30,9 +30,9 @@ import type { TagEntity } from '@db/entities/TagEntity'; import type { Variables } from '@db/entities/Variables'; import type { SourceControlWorkflowVersionId } from './types/sourceControlWorkflowVersionId'; import type { ExportableCredential } from './types/exportableCredential'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; import { TagRepository } from '@db/repositories/tag.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ApplicationError } from 'n8n-workflow'; diff --git a/packages/cli/src/environments/sourceControl/sourceControlExport.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlExport.service.ee.ts index e586357093..6d2d468e30 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlExport.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlExport.service.ee.ts @@ -25,7 +25,7 @@ import type { SourceControlledFile } from './types/sourceControlledFile'; import { VariablesService } from '../variables/variables.service.ee'; import { TagRepository } from '@db/repositories/tag.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { WorkflowTagMappingRepository } from '@db/repositories/workflowTagMapping.repository'; @@ -293,11 +293,18 @@ export class SourceControlExportService { }; } + /** + * Edge case: Do not export `oauthTokenData`, so that that the + * pulling instance reconnects instead of trying to use stubbed values. + */ + const credentialData = credentials.getData(); + const { oauthTokenData, ...rest } = credentialData; + const stub: ExportableCredential = { id, name, type, - data: this.replaceCredentialData(credentials.getData()), + data: this.replaceCredentialData(rest), ownedBy: owner, }; diff --git a/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts index 1904761996..91220340b7 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts @@ -20,7 +20,7 @@ import { } from './constants'; import { sourceControlFoldersExistCheck } from './sourceControlHelper.ee'; import type { User } from '@db/entities/User'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { ApplicationError } from 'n8n-workflow'; import { OwnershipService } from '@/services/ownership.service'; import { SourceControlPreferencesService } from './sourceControlPreferences.service.ee'; diff --git a/packages/cli/src/environments/sourceControl/sourceControlHelper.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlHelper.ee.ts index 8610f03015..65cd88bfa5 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlHelper.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlHelper.ee.ts @@ -1,5 +1,5 @@ import { Container } from 'typedi'; -import { License } from '@/License'; +import { License } from '@/license'; import { generateKeyPairSync } from 'crypto'; import type { KeyPair } from './types/keyPair'; import { constants as fsConstants, mkdirSync, accessSync } from 'fs'; @@ -11,7 +11,7 @@ import { import type { SourceControlledFile } from './types/sourceControlledFile'; import path from 'path'; import type { KeyPairType } from './types/keyPairType'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; export function stringContainsExpression(testString: string): boolean { return /^=.*\{\{.*\}\}/.test(testString); diff --git a/packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts index d9a366b6f0..bd04937bc1 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts @@ -17,17 +17,17 @@ import type { Variables } from '@db/entities/Variables'; import { SharedCredentials } from '@db/entities/SharedCredentials'; import type { WorkflowTagMapping } from '@db/entities/WorkflowTagMapping'; import type { TagEntity } from '@db/entities/TagEntity'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In } from '@n8n/typeorm'; -import { isUniqueConstraintError } from '@/ResponseHelper'; +import { isUniqueConstraintError } from '@/response-helper'; import type { SourceControlWorkflowVersionId } from './types/sourceControlWorkflowVersionId'; import { getCredentialExportPath, getWorkflowExportPath } from './sourceControlHelper.ee'; import type { SourceControlledFile } from './types/sourceControlledFile'; import { VariablesService } from '../variables/variables.service.ee'; import { TagRepository } from '@db/repositories/tag.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; @@ -326,7 +326,12 @@ export class SourceControlImportService { if (existingCredential?.data) { newCredentialObject.data = existingCredential.data; } else { - newCredentialObject.setData(data); + /** + * Edge case: Do not import `oauthTokenData`, so that that the + * pulling instance reconnects instead of trying to use stubbed values. + */ + const { oauthTokenData, ...rest } = data; + newCredentialObject.setData(rest); } this.logger.debug(`Updating credential id ${newCredentialObject.id as string}`); diff --git a/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts index f33cfc2dc6..52a2f6192d 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts @@ -16,7 +16,7 @@ import { import path from 'path'; import type { KeyPairType } from './types/keyPairType'; import config from '@/config'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { SettingsRepository } from '@db/repositories/settings.repository'; @Service() diff --git a/packages/cli/src/environments/variables/environmentHelpers.ts b/packages/cli/src/environments/variables/environmentHelpers.ts index 7868f22715..9a3f2362f8 100644 --- a/packages/cli/src/environments/variables/environmentHelpers.ts +++ b/packages/cli/src/environments/variables/environmentHelpers.ts @@ -1,5 +1,5 @@ import { Container } from 'typedi'; -import { License } from '@/License'; +import { License } from '@/license'; export function isVariablesEnabled(): boolean { const license = Container.get(License); diff --git a/packages/cli/src/environments/variables/variables.service.ee.ts b/packages/cli/src/environments/variables/variables.service.ee.ts index 94233da065..78a3d23fbe 100644 --- a/packages/cli/src/environments/variables/variables.service.ee.ts +++ b/packages/cli/src/environments/variables/variables.service.ee.ts @@ -6,7 +6,7 @@ import { CacheService } from '@/services/cache/cache.service'; import { VariablesRepository } from '@db/repositories/variables.repository'; import { VariableCountLimitReachedError } from '@/errors/variable-count-limit-reached.error'; import { VariableValidationError } from '@/errors/variable-validation.error'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @Service() export class VariablesService { diff --git a/packages/cli/src/ErrorReporting.ts b/packages/cli/src/error-reporting.ts similarity index 87% rename from packages/cli/src/ErrorReporting.ts rename to packages/cli/src/error-reporting.ts index 33a1da0c39..a8b25c891d 100644 --- a/packages/cli/src/ErrorReporting.ts +++ b/packages/cli/src/error-reporting.ts @@ -1,6 +1,8 @@ import { createHash } from 'crypto'; import config from '@/config'; import { ErrorReporterProxy, ApplicationError } from 'n8n-workflow'; +// eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import +import { QueryFailedError } from '@n8n/typeorm'; let initialized = false; @@ -65,6 +67,13 @@ export const initErrorHandling = async () => { addEventProcessor((event, { originalException }) => { if (!originalException) return null; + if ( + originalException instanceof QueryFailedError && + ['SQLITE_FULL', 'SQLITE_IOERR'].some((errMsg) => originalException.message.includes(errMsg)) + ) { + return null; + } + if (originalException instanceof ApplicationError) { const { level, extra, tags } = originalException; if (level === 'warning') return null; diff --git a/packages/cli/src/errors/max-stalled-count.error.ts b/packages/cli/src/errors/max-stalled-count.error.ts index 6715de0ade..653ca18eac 100644 --- a/packages/cli/src/errors/max-stalled-count.error.ts +++ b/packages/cli/src/errors/max-stalled-count.error.ts @@ -1,7 +1,7 @@ import { ApplicationError } from 'n8n-workflow'; /** - * See https://github.com/OptimalBits/bull/blob/60fa88f08637f0325639988a3f054880a04ce402/docs/README.md?plain=1#L133 + * @docs https://docs.bullmq.io/guide/workers/stalled-jobs */ export class MaxStalledCountError extends ApplicationError { constructor(cause: Error) { diff --git a/packages/cli/src/errors/redactable.error.ts b/packages/cli/src/errors/redactable.error.ts index 0f6697a065..0d5b07ac50 100644 --- a/packages/cli/src/errors/redactable.error.ts +++ b/packages/cli/src/errors/redactable.error.ts @@ -3,7 +3,7 @@ import { ApplicationError } from 'n8n-workflow'; export class RedactableError extends ApplicationError { constructor(fieldName: string, args: string) { super( - `Failed to find "${fieldName}" property in argument "${args.toString()}". Please set the decorator \`@Redactable()\` only on \`AuditEventRelay\` methods where the argument contains a "${fieldName}" property.`, + `Failed to find "${fieldName}" property in argument "${args.toString()}". Please set the decorator \`@Redactable()\` only on \`LogStreamingEventRelay\` methods where the argument contains a "${fieldName}" property.`, ); } } diff --git a/packages/cli/test/unit/utils.test.ts b/packages/cli/src/errors/response-errors/__tests__/webhook-not-found.error.test.ts similarity index 100% rename from packages/cli/test/unit/utils.test.ts rename to packages/cli/src/errors/response-errors/__tests__/webhook-not-found.error.test.ts diff --git a/packages/cli/src/errors/response-errors/invalid-mfa-code.error.ts b/packages/cli/src/errors/response-errors/invalid-mfa-code.error.ts new file mode 100644 index 0000000000..cc00976a84 --- /dev/null +++ b/packages/cli/src/errors/response-errors/invalid-mfa-code.error.ts @@ -0,0 +1,7 @@ +import { ForbiddenError } from './forbidden.error'; + +export class InvalidMfaCodeError extends ForbiddenError { + constructor(hint?: string) { + super('Invalid two-factor code.', hint); + } +} diff --git a/packages/cli/src/errors/subworkflow-policy-denial.error.ts b/packages/cli/src/errors/subworkflow-policy-denial.error.ts new file mode 100644 index 0000000000..03149a6f11 --- /dev/null +++ b/packages/cli/src/errors/subworkflow-policy-denial.error.ts @@ -0,0 +1,25 @@ +import { WorkflowOperationError } from 'n8n-workflow'; +import type { Project } from '@/databases/entities/Project'; +import type { INode } from 'n8n-workflow'; + +type SubworkflowPolicyDenialErrorParams = { + subworkflowId: string; + subworkflowProject: Project; + areOwnedBySameProject?: boolean; + node?: INode; +}; + +export class SubworkflowPolicyDenialError extends WorkflowOperationError { + constructor({ + subworkflowId, + subworkflowProject, + areOwnedBySameProject, + node, + }: SubworkflowPolicyDenialErrorParams) { + const description = areOwnedBySameProject + ? 'Change the settings of the sub-workflow so it can be called by this one.' + : `An admin for the ${subworkflowProject.name} project can make this change. You may need to tell them the ID of the sub-workflow, which is ${subworkflowId}`; + + super(`Target workflow ID ${subworkflowId} may not be called`, node, description); + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/index.ts b/packages/cli/src/eventbus/EventMessageClasses/index.ts index 461394d378..2e4d78560d 100644 --- a/packages/cli/src/eventbus/EventMessageClasses/index.ts +++ b/packages/cli/src/eventbus/EventMessageClasses/index.ts @@ -10,7 +10,6 @@ export const eventNamesWorkflow = [ 'n8n.workflow.started', 'n8n.workflow.success', 'n8n.workflow.failed', - 'n8n.workflow.crashed', ] as const; export const eventNamesGeneric = ['n8n.worker.started', 'n8n.worker.stopped'] as const; export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const; diff --git a/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts index 758eeb5ae5..38ec4e8dc7 100644 --- a/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts +++ b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts @@ -12,7 +12,7 @@ import { EventDestinationsRepository } from '@db/repositories/eventDestinations. import { ExecutionRepository } from '@db/repositories/execution.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { OrchestrationService } from '@/services/orchestration.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { EventMessageTypes } from '../EventMessageClasses/'; import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee'; @@ -34,7 +34,7 @@ import { EventMessageAiNode, type EventMessageAiNodeOptions, } from '../EventMessageClasses/EventMessageAiNode'; -import { License } from '@/License'; +import { License } from '@/license'; import type { EventMessageExecutionOptions } from '../EventMessageClasses/EventMessageExecution'; import { EventMessageExecution } from '../EventMessageClasses/EventMessageExecution'; import { GlobalConfig } from '@n8n/config'; @@ -52,6 +52,8 @@ export interface MessageEventBusInitializeOptions { } @Service() +// TODO: Convert to TypedEventEmitter +// eslint-disable-next-line n8n-local-rules/no-type-unsafe-event-emitter export class MessageEventBus extends EventEmitter { private isInitialized = false; diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts index 6a7fef6d6e..3cd86f4368 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts @@ -2,13 +2,13 @@ import { v4 as uuid } from 'uuid'; import { Container } from 'typedi'; import type { INodeCredentials, MessageEventBusDestinationOptions } from 'n8n-workflow'; import { MessageEventBusDestinationTypeNames } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage'; import type { EventMessageTypes } from '../EventMessageClasses'; import type { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm'; import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus'; import { EventDestinationsRepository } from '@db/repositories/eventDestinations.repository'; -import { License } from '@/License'; +import { License } from '@/license'; export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions { // Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please. diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationFromDb.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationFromDb.ts index 27951448ce..886d1e35dd 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationFromDb.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationFromDb.ts @@ -6,7 +6,7 @@ import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSe import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee'; import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee'; import { Container } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; export function messageEventBusDestinationFromDb( eventBusInstance: MessageEventBus, diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts index f57705319c..8d9b743829 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts @@ -10,7 +10,7 @@ import { MessageEventBusDestination } from './MessageEventBusDestination.ee'; import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus'; import Container from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; export const isMessageEventBusDestinationSyslogOptions = ( candidate: unknown, ): candidate is MessageEventBusDestinationSyslogOptions => { diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts index 20f7dbb411..5521854952 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts @@ -11,11 +11,11 @@ import type { IWorkflowExecuteAdditionalData, MessageEventBusDestinationWebhookOptions, } from 'n8n-workflow'; -import { CredentialsHelper } from '@/CredentialsHelper'; +import { CredentialsHelper } from '@/credentials-helper'; import { Agent as HTTPSAgent } from 'https'; import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus'; -import * as SecretsHelpers from '@/ExternalSecrets/externalSecretsHelper.ee'; +import * as SecretsHelpers from '@/external-secrets/external-secrets-helper.ee'; import Container from 'typedi'; export const isMessageEventBusDestinationWebhookOptions = ( diff --git a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts index 08899c0e09..617dacd283 100644 --- a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts +++ b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts @@ -26,7 +26,7 @@ import { } from '../EventMessageClasses/EventMessageConfirm'; import { once as eventOnce } from 'events'; import { inTest } from '@/constants'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import Container from 'typedi'; import { GlobalConfig } from '@n8n/config'; @@ -225,7 +225,6 @@ export class MessageEventBusLogWriter { break; case 'n8n.workflow.success': case 'n8n.workflow.failed': - case 'n8n.workflow.crashed': case 'n8n.execution.throttled': case 'n8n.execution.started-during-bootup': delete results.unfinishedExecutions[executionId]; diff --git a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts index 4686a1cf3c..69d2e8ce26 100644 --- a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts +++ b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts @@ -6,7 +6,6 @@ import type { MessageEventBusLogWriterOptions } from './MessageEventBusLogWriter let logFileBasePath = ''; let loggingPaused = true; let keepFiles = 10; -let fileStatTimer: NodeJS.Timer; let maxLogFileSizeInKB = 102400; function setLogFileBasePath(basePath: string) { @@ -117,7 +116,7 @@ if (!isMainThread) { if (logFileBasePath) { renameAndCreateLogs(); loggingPaused = false; - fileStatTimer = setInterval(async () => { + setInterval(async () => { await checkFileSize(buildLogFileNameWithCounter()); }, 5000); } diff --git a/packages/cli/src/eventbus/__tests__/audit-event-relay.service.test.ts b/packages/cli/src/eventbus/__tests__/audit-event-relay.service.test.ts deleted file mode 100644 index 8039220607..0000000000 --- a/packages/cli/src/eventbus/__tests__/audit-event-relay.service.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { mock } from 'jest-mock-extended'; -import { AuditEventRelay } from '../audit-event-relay.service'; -import type { MessageEventBus } from '../MessageEventBus/MessageEventBus'; -import type { Event } from '../event.types'; -import type { EventService } from '../event.service'; - -describe('AuditorService', () => { - const eventBus = mock(); - const eventService = mock(); - const auditor = new AuditEventRelay(eventService, eventBus); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should handle `user-deleted` event', () => { - const arg: Event['user-deleted'] = { - user: { - id: '123', - email: 'john@n8n.io', - firstName: 'John', - lastName: 'Doe', - role: 'some-role', - }, - }; - - // @ts-expect-error Private method - auditor.userDeleted(arg); - - expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ - eventName: 'n8n.audit.user.deleted', - payload: { - userId: '123', - _email: 'john@n8n.io', - _firstName: 'John', - _lastName: 'Doe', - globalRole: 'some-role', - }, - }); - }); - - it('should handle `user-invite-email-click` event', () => { - const arg: Event['user-invite-email-click'] = { - inviter: { - id: '123', - email: 'john@n8n.io', - firstName: 'John', - lastName: 'Doe', - role: 'some-role', - }, - invitee: { - id: '456', - email: 'jane@n8n.io', - firstName: 'Jane', - lastName: 'Doe', - role: 'some-other-role', - }, - }; - - // @ts-expect-error Private method - auditor.userInviteEmailClick(arg); - - expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ - eventName: 'n8n.audit.user.invitation.accepted', - payload: { - inviter: { - userId: '123', - _email: 'john@n8n.io', - _firstName: 'John', - _lastName: 'Doe', - globalRole: 'some-role', - }, - invitee: { - userId: '456', - _email: 'jane@n8n.io', - _firstName: 'Jane', - _lastName: 'Doe', - globalRole: 'some-other-role', - }, - }, - }); - }); -}); diff --git a/packages/cli/src/eventbus/audit-event-relay.service.ts b/packages/cli/src/eventbus/audit-event-relay.service.ts deleted file mode 100644 index 9c73494520..0000000000 --- a/packages/cli/src/eventbus/audit-event-relay.service.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { Service } from 'typedi'; -import { MessageEventBus } from './MessageEventBus/MessageEventBus'; -import { Redactable } from '@/decorators/Redactable'; -import { EventService } from './event.service'; -import type { Event } from './event.types'; -import type { IWorkflowBase } from 'n8n-workflow'; - -@Service() -export class AuditEventRelay { - constructor( - private readonly eventService: EventService, - private readonly eventBus: MessageEventBus, - ) {} - - init() { - this.setupHandlers(); - } - - private setupHandlers() { - this.eventService.on('workflow-created', (event) => this.workflowCreated(event)); - this.eventService.on('workflow-deleted', (event) => this.workflowDeleted(event)); - this.eventService.on('workflow-saved', (event) => this.workflowSaved(event)); - this.eventService.on('workflow-pre-execute', (event) => this.workflowPreExecute(event)); - this.eventService.on('workflow-post-execute', (event) => this.workflowPostExecute(event)); - this.eventService.on('node-pre-execute', (event) => this.nodePreExecute(event)); - this.eventService.on('node-post-execute', (event) => this.nodePostExecute(event)); - this.eventService.on('user-deleted', (event) => this.userDeleted(event)); - this.eventService.on('user-invited', (event) => this.userInvited(event)); - this.eventService.on('user-reinvited', (event) => this.userReinvited(event)); - this.eventService.on('user-updated', (event) => this.userUpdated(event)); - this.eventService.on('user-signed-up', (event) => this.userSignedUp(event)); - this.eventService.on('user-logged-in', (event) => this.userLoggedIn(event)); - this.eventService.on('user-login-failed', (event) => this.userLoginFailed(event)); - this.eventService.on('user-invite-email-click', (event) => this.userInviteEmailClick(event)); - this.eventService.on('user-password-reset-email-click', (event) => - this.userPasswordResetEmailClick(event), - ); - this.eventService.on('user-password-reset-request-click', (event) => - this.userPasswordResetRequestClick(event), - ); - this.eventService.on('public-api-key-created', (event) => this.publicApiKeyCreated(event)); - this.eventService.on('public-api-key-deleted', (event) => this.publicApiKeyDeleted(event)); - this.eventService.on('email-failed', (event) => this.emailFailed(event)); - this.eventService.on('credentials-created', (event) => this.credentialsCreated(event)); - this.eventService.on('credentials-deleted', (event) => this.credentialsDeleted(event)); - this.eventService.on('credentials-shared', (event) => this.credentialsShared(event)); - this.eventService.on('credentials-updated', (event) => this.credentialsUpdated(event)); - this.eventService.on('credentials-deleted', (event) => this.credentialsDeleted(event)); - this.eventService.on('community-package-installed', (event) => - this.communityPackageInstalled(event), - ); - this.eventService.on('community-package-updated', (event) => - this.communityPackageUpdated(event), - ); - this.eventService.on('community-package-deleted', (event) => - this.communityPackageDeleted(event), - ); - this.eventService.on('execution-throttled', (event) => this.executionThrottled(event)); - this.eventService.on('execution-started-during-bootup', (event) => - this.executionStartedDuringBootup(event), - ); - } - - /** - * Workflow - */ - - @Redactable() - private workflowCreated({ user, workflow }: Event['workflow-created']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.workflow.created', - payload: { - ...user, - workflowId: workflow.id, - workflowName: workflow.name, - }, - }); - } - - @Redactable() - private workflowDeleted({ user, workflowId }: Event['workflow-deleted']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.workflow.deleted', - payload: { ...user, workflowId }, - }); - } - - @Redactable() - private workflowSaved({ user, workflowId, workflowName }: Event['workflow-saved']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.workflow.updated', - payload: { - ...user, - workflowId, - workflowName, - }, - }); - } - - private workflowPreExecute({ data, executionId }: Event['workflow-pre-execute']) { - const payload = - 'executionData' in data - ? { - executionId, - userId: data.userId, - workflowId: data.workflowData.id, - isManual: data.executionMode === 'manual', - workflowName: data.workflowData.name, - } - : { - executionId, - userId: undefined, - workflowId: (data as IWorkflowBase).id, - isManual: false, - workflowName: (data as IWorkflowBase).name, - }; - - void this.eventBus.sendWorkflowEvent({ - eventName: 'n8n.workflow.started', - payload, - }); - } - - private workflowPostExecute(event: Event['workflow-post-execute']) { - void this.eventBus.sendWorkflowEvent({ - eventName: 'n8n.workflow.success', - payload: event, - }); - } - - /** - * Node - */ - - private nodePreExecute({ workflow, executionId, nodeName }: Event['node-pre-execute']) { - void this.eventBus.sendNodeEvent({ - eventName: 'n8n.node.started', - payload: { - workflowId: workflow.id, - workflowName: workflow.name, - executionId, - nodeType: workflow.nodes.find((n) => n.name === nodeName)?.type, - nodeName, - }, - }); - } - - private nodePostExecute({ workflow, executionId, nodeName }: Event['node-post-execute']) { - void this.eventBus.sendNodeEvent({ - eventName: 'n8n.node.finished', - payload: { - workflowId: workflow.id, - workflowName: workflow.name, - executionId, - nodeType: workflow.nodes.find((n) => n.name === nodeName)?.type, - nodeName, - }, - }); - } - - /** - * User - */ - - @Redactable() - private userDeleted({ user }: Event['user-deleted']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.deleted', - payload: user, - }); - } - - @Redactable() - private userInvited({ user, targetUserId }: Event['user-invited']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.invited', - payload: { ...user, targetUserId }, - }); - } - - @Redactable() - private userReinvited({ user, targetUserId }: Event['user-reinvited']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.reinvited', - payload: { ...user, targetUserId }, - }); - } - - @Redactable() - private userUpdated({ user, fieldsChanged }: Event['user-updated']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.updated', - payload: { ...user, fieldsChanged }, - }); - } - - /** - * Auth - */ - - @Redactable() - private userSignedUp({ user }: Event['user-signed-up']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.signedup', - payload: user, - }); - } - - @Redactable() - private userLoggedIn({ user, authenticationMethod }: Event['user-logged-in']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.login.success', - payload: { ...user, authenticationMethod }, - }); - } - - private userLoginFailed( - event: Event['user-login-failed'] /* exception: no `UserLike` to redact */, - ) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.login.failed', - payload: event, - }); - } - - /** - * Click - */ - - @Redactable('inviter') - @Redactable('invitee') - private userInviteEmailClick(event: Event['user-invite-email-click']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.invitation.accepted', - payload: event, - }); - } - - @Redactable() - private userPasswordResetEmailClick({ user }: Event['user-password-reset-email-click']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.reset', - payload: user, - }); - } - - @Redactable() - private userPasswordResetRequestClick({ user }: Event['user-password-reset-request-click']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.reset.requested', - payload: user, - }); - } - - /** - * API key - */ - - @Redactable() - private publicApiKeyCreated({ user }: Event['public-api-key-created']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.api.created', - payload: user, - }); - } - - @Redactable() - private publicApiKeyDeleted({ user }: Event['public-api-key-deleted']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.api.deleted', - payload: user, - }); - } - - /** - * Emailing - */ - - @Redactable() - private emailFailed({ user, messageType }: Event['email-failed']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.email.failed', - payload: { ...user, messageType }, - }); - } - - /** - * Credentials - */ - - @Redactable() - private credentialsCreated({ user, ...rest }: Event['credentials-created']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.credentials.created', - payload: { ...user, ...rest }, - }); - } - - @Redactable() - private credentialsDeleted({ user, ...rest }: Event['credentials-deleted']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.credentials.deleted', - payload: { ...user, ...rest }, - }); - } - - @Redactable() - private credentialsShared({ user, ...rest }: Event['credentials-shared']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.credentials.shared', - payload: { ...user, ...rest }, - }); - } - - @Redactable() - private credentialsUpdated({ user, ...rest }: Event['credentials-updated']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.user.credentials.updated', - payload: { ...user, ...rest }, - }); - } - - /** - * Community package - */ - - @Redactable() - private communityPackageInstalled({ user, ...rest }: Event['community-package-installed']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.package.installed', - payload: { ...user, ...rest }, - }); - } - - @Redactable() - private communityPackageUpdated({ user, ...rest }: Event['community-package-updated']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.package.updated', - payload: { ...user, ...rest }, - }); - } - - @Redactable() - private communityPackageDeleted({ user, ...rest }: Event['community-package-deleted']) { - void this.eventBus.sendAuditEvent({ - eventName: 'n8n.audit.package.deleted', - payload: { ...user, ...rest }, - }); - } - - /** - * Execution - */ - - private executionThrottled({ executionId }: Event['execution-throttled']) { - void this.eventBus.sendExecutionEvent({ - eventName: 'n8n.execution.throttled', - payload: { executionId }, - }); - } - - private executionStartedDuringBootup({ executionId }: Event['execution-started-during-bootup']) { - void this.eventBus.sendExecutionEvent({ - eventName: 'n8n.execution.started-during-bootup', - payload: { executionId }, - }); - } -} diff --git a/packages/cli/src/eventbus/event.service.ts b/packages/cli/src/eventbus/event.service.ts deleted file mode 100644 index 2df51af22c..0000000000 --- a/packages/cli/src/eventbus/event.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { EventEmitter } from 'node:events'; -import { Service } from 'typedi'; -import type { Event } from './event.types'; - -@Service() -export class EventService extends EventEmitter { - emit(eventName: K, arg?: Event[K]) { - super.emit(eventName, arg); - return true; - } - - on(eventName: K, handler: (arg: Event[K]) => void) { - super.on(eventName, handler); - return this; - } -} diff --git a/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts new file mode 100644 index 0000000000..720e2d494c --- /dev/null +++ b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts @@ -0,0 +1,948 @@ +import { mock } from 'jest-mock-extended'; +import { LogStreamingEventRelay } from '@/events/log-streaming-event-relay'; +import { EventService } from '@/events/event.service'; +import type { INode, IRun, IWorkflowBase } from 'n8n-workflow'; +import type { IWorkflowDb } from '@/Interfaces'; +import type { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; +import type { RelayEventMap } from '@/events/relay-event-map'; + +describe('LogStreamingEventRelay', () => { + const eventBus = mock(); + const eventService = new EventService(); + new LogStreamingEventRelay(eventService, eventBus).init(); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('workflow events', () => { + it('should log on `workflow-created` event', () => { + const event: RelayEventMap['workflow-created'] = { + user: { + id: '123', + email: 'john@n8n.io', + firstName: 'John', + lastName: 'Doe', + role: 'owner', + }, + workflow: mock({ + id: 'wf123', + name: 'Test Workflow', + }), + publicApi: false, + projectId: 'proj123', + projectType: 'personal', + }; + + eventService.emit('workflow-created', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.workflow.created', + payload: { + userId: '123', + _email: 'john@n8n.io', + _firstName: 'John', + _lastName: 'Doe', + globalRole: 'owner', + workflowId: 'wf123', + workflowName: 'Test Workflow', + }, + }); + }); + + it('should log on `workflow-deleted` event', () => { + const event: RelayEventMap['workflow-deleted'] = { + user: { + id: '456', + email: 'jane@n8n.io', + firstName: 'Jane', + lastName: 'Smith', + role: 'user', + }, + workflowId: 'wf789', + publicApi: false, + }; + + eventService.emit('workflow-deleted', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.workflow.deleted', + payload: { + userId: '456', + _email: 'jane@n8n.io', + _firstName: 'Jane', + _lastName: 'Smith', + globalRole: 'user', + workflowId: 'wf789', + }, + }); + }); + + it('should log on `workflow-saved` event', () => { + const event: RelayEventMap['workflow-saved'] = { + user: { + id: '789', + email: 'alex@n8n.io', + firstName: 'Alex', + lastName: 'Johnson', + role: 'editor', + }, + workflow: mock({ id: 'wf101', name: 'Updated Workflow' }), + publicApi: false, + }; + + eventService.emit('workflow-saved', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.workflow.updated', + payload: { + userId: '789', + _email: 'alex@n8n.io', + _firstName: 'Alex', + _lastName: 'Johnson', + globalRole: 'editor', + workflowId: 'wf101', + workflowName: 'Updated Workflow', + }, + }); + }); + + it('should log on `workflow-pre-execute` event', () => { + const workflow = mock({ + id: 'wf202', + name: 'Test Workflow', + active: true, + nodes: [], + connections: {}, + staticData: undefined, + settings: {}, + }); + + const event: RelayEventMap['workflow-pre-execute'] = { + executionId: 'exec123', + data: workflow, + }; + + eventService.emit('workflow-pre-execute', event); + + expect(eventBus.sendWorkflowEvent).toHaveBeenCalledWith({ + eventName: 'n8n.workflow.started', + payload: { + executionId: 'exec123', + userId: undefined, + workflowId: 'wf202', + isManual: false, + workflowName: 'Test Workflow', + }, + }); + }); + + it('should log on `workflow-post-execute` for successful execution', () => { + const payload = mock({ + executionId: 'some-id', + userId: 'some-id', + workflow: mock({ id: 'some-id', name: 'some-name' }), + runData: mock({ + finished: true, + status: 'success', + mode: 'manual', + data: { resultData: {} }, + }), + }); + + eventService.emit('workflow-post-execute', payload); + + const { runData: _, workflow: __, ...rest } = payload; + + expect(eventBus.sendWorkflowEvent).toHaveBeenCalledWith({ + eventName: 'n8n.workflow.success', + payload: { + ...rest, + success: true, // same as finished + isManual: true, + workflowName: 'some-name', + workflowId: 'some-id', + }, + }); + }); + + it('should log on `workflow-post-execute` event for failed execution', () => { + const runData = mock({ + status: 'error', + mode: 'manual', + finished: false, + data: { + resultData: { + lastNodeExecuted: 'some-node', + // @ts-expect-error Partial mock + error: { + node: mock({ type: 'some-type' }), + message: 'some-message', + }, + errorMessage: 'some-message', + }, + }, + }) as unknown as IRun; + + const event = { + executionId: 'some-id', + userId: 'some-id', + workflow: mock({ id: 'some-id', name: 'some-name' }), + runData, + }; + + eventService.emit('workflow-post-execute', event); + + const { runData: _, workflow: __, ...rest } = event; + + expect(eventBus.sendWorkflowEvent).toHaveBeenCalledWith({ + eventName: 'n8n.workflow.failed', + payload: { + ...rest, + success: false, // same as finished + isManual: true, + workflowName: 'some-name', + workflowId: 'some-id', + lastNodeExecuted: 'some-node', + errorNodeType: 'some-type', + errorMessage: 'some-message', + }, + }); + }); + }); + + describe('user events', () => { + it('should log on `user-updated` event', () => { + const event: RelayEventMap['user-updated'] = { + user: { + id: 'user456', + email: 'updated@example.com', + firstName: 'Updated', + lastName: 'User', + role: 'global:member', + }, + fieldsChanged: ['firstName', 'lastName', 'password'], + }; + + eventService.emit('user-updated', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.updated', + payload: { + userId: 'user456', + _email: 'updated@example.com', + _firstName: 'Updated', + _lastName: 'User', + globalRole: 'global:member', + fieldsChanged: ['firstName', 'lastName', 'password'], + }, + }); + }); + + it('should log on `user-deleted` event', () => { + const event: RelayEventMap['user-deleted'] = { + user: { + id: '123', + email: 'john@n8n.io', + firstName: 'John', + lastName: 'Doe', + role: 'some-role', + }, + targetUserOldStatus: 'active', + publicApi: false, + migrationStrategy: 'transfer_data', + targetUserId: '456', + migrationUserId: '789', + }; + + eventService.emit('user-deleted', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.deleted', + payload: { + userId: '123', + _email: 'john@n8n.io', + _firstName: 'John', + _lastName: 'Doe', + globalRole: 'some-role', + }, + }); + }); + + it('should log on `user-invited` event', () => { + const event: RelayEventMap['user-invited'] = { + user: { + id: 'user101', + email: 'inviter@example.com', + firstName: 'Inviter', + lastName: 'User', + role: 'global:owner', + }, + targetUserId: ['newUser123'], + publicApi: false, + emailSent: true, + inviteeRole: 'global:member', + }; + + eventService.emit('user-invited', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.invited', + payload: { + userId: 'user101', + _email: 'inviter@example.com', + _firstName: 'Inviter', + _lastName: 'User', + globalRole: 'global:owner', + targetUserId: ['newUser123'], + }, + }); + }); + + it('should log on `user-reinvited` event', () => { + const event: RelayEventMap['user-reinvited'] = { + user: { + id: 'user202', + email: 'reinviter@example.com', + firstName: 'Reinviter', + lastName: 'User', + role: 'global:admin', + }, + targetUserId: ['existingUser456'], + }; + + eventService.emit('user-reinvited', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.reinvited', + payload: { + userId: 'user202', + _email: 'reinviter@example.com', + _firstName: 'Reinviter', + _lastName: 'User', + globalRole: 'global:admin', + targetUserId: ['existingUser456'], + }, + }); + }); + + it('should log on `user-signed-up` event', () => { + const event: RelayEventMap['user-signed-up'] = { + user: { + id: 'user303', + email: 'newuser@example.com', + firstName: 'New', + lastName: 'User', + role: 'global:member', + }, + userType: 'email', + wasDisabledLdapUser: false, + }; + + eventService.emit('user-signed-up', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.signedup', + payload: { + userId: 'user303', + _email: 'newuser@example.com', + _firstName: 'New', + _lastName: 'User', + globalRole: 'global:member', + }, + }); + }); + + it('should log on `user-logged-in` event', () => { + const event: RelayEventMap['user-logged-in'] = { + user: { + id: 'user404', + email: 'loggedin@example.com', + firstName: 'Logged', + lastName: 'In', + role: 'global:owner', + }, + authenticationMethod: 'email', + }; + + eventService.emit('user-logged-in', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.login.success', + payload: { + userId: 'user404', + _email: 'loggedin@example.com', + _firstName: 'Logged', + _lastName: 'In', + globalRole: 'global:owner', + authenticationMethod: 'email', + }, + }); + }); + }); + + describe('click events', () => { + it('should log on `user-password-reset-request-click` event', () => { + const event: RelayEventMap['user-password-reset-request-click'] = { + user: { + id: 'user101', + email: 'user101@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:member', + }, + }; + + eventService.emit('user-password-reset-request-click', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.reset.requested', + payload: { + userId: 'user101', + _email: 'user101@example.com', + _firstName: 'John', + _lastName: 'Doe', + globalRole: 'global:member', + }, + }); + }); + + it('should log on `user-invite-email-click` event', () => { + const event: RelayEventMap['user-invite-email-click'] = { + inviter: { + id: '123', + email: 'john@n8n.io', + firstName: 'John', + lastName: 'Doe', + role: 'some-role', + }, + invitee: { + id: '456', + email: 'jane@n8n.io', + firstName: 'Jane', + lastName: 'Doe', + role: 'some-other-role', + }, + }; + + eventService.emit('user-invite-email-click', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.invitation.accepted', + payload: { + inviter: { + userId: '123', + _email: 'john@n8n.io', + _firstName: 'John', + _lastName: 'Doe', + globalRole: 'some-role', + }, + invitee: { + userId: '456', + _email: 'jane@n8n.io', + _firstName: 'Jane', + _lastName: 'Doe', + globalRole: 'some-other-role', + }, + }, + }); + }); + + it('should log on `user-password-reset-email-click` event', () => { + const event: RelayEventMap['user-password-reset-email-click'] = { + user: { + id: 'user505', + email: 'resetuser@example.com', + firstName: 'Reset', + lastName: 'User', + role: 'global:member', + }, + }; + + eventService.emit('user-password-reset-email-click', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.reset', + payload: { + userId: 'user505', + _email: 'resetuser@example.com', + _firstName: 'Reset', + _lastName: 'User', + globalRole: 'global:member', + }, + }); + }); + }); + + describe('node events', () => { + it('should log on `node-pre-execute` event', () => { + const workflow = mock({ + id: 'wf303', + name: 'Test Workflow with Nodes', + active: true, + nodes: [ + { + id: 'node1', + name: 'Start Node', + type: 'n8n-nodes-base.start', + typeVersion: 1, + position: [100, 200], + }, + { + id: 'node2', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [300, 200], + }, + ], + connections: {}, + settings: {}, + }); + + const event: RelayEventMap['node-pre-execute'] = { + executionId: 'exec456', + nodeName: 'HTTP Request', + workflow, + }; + + eventService.emit('node-pre-execute', event); + + expect(eventBus.sendNodeEvent).toHaveBeenCalledWith({ + eventName: 'n8n.node.started', + payload: { + executionId: 'exec456', + nodeName: 'HTTP Request', + workflowId: 'wf303', + workflowName: 'Test Workflow with Nodes', + nodeType: 'n8n-nodes-base.httpRequest', + }, + }); + }); + + it('should log on `node-post-execute` event', () => { + const workflow = mock({ + id: 'wf404', + name: 'Test Workflow with Completed Node', + active: true, + nodes: [ + { + id: 'node1', + name: 'Start Node', + type: 'n8n-nodes-base.start', + typeVersion: 1, + position: [100, 200], + }, + { + id: 'node2', + name: 'HTTP Response', + type: 'n8n-nodes-base.httpResponse', + typeVersion: 1, + position: [300, 200], + }, + ], + connections: {}, + settings: {}, + }); + + const event: RelayEventMap['node-post-execute'] = { + executionId: 'exec789', + nodeName: 'HTTP Response', + workflow, + }; + + eventService.emit('node-post-execute', event); + + expect(eventBus.sendNodeEvent).toHaveBeenCalledWith({ + eventName: 'n8n.node.finished', + payload: { + executionId: 'exec789', + nodeName: 'HTTP Response', + workflowId: 'wf404', + workflowName: 'Test Workflow with Completed Node', + nodeType: 'n8n-nodes-base.httpResponse', + }, + }); + }); + }); + + describe('credentials events', () => { + it('should log on `credentials-shared` event', () => { + const event: RelayEventMap['credentials-shared'] = { + user: { + id: 'user123', + email: 'sharer@example.com', + firstName: 'Alice', + lastName: 'Sharer', + role: 'global:owner', + }, + credentialId: 'cred789', + credentialType: 'githubApi', + userIdSharer: 'user123', + userIdsShareesAdded: ['user456', 'user789'], + shareesRemoved: null, + }; + + eventService.emit('credentials-shared', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.credentials.shared', + payload: { + userId: 'user123', + _email: 'sharer@example.com', + _firstName: 'Alice', + _lastName: 'Sharer', + globalRole: 'global:owner', + credentialId: 'cred789', + credentialType: 'githubApi', + userIdSharer: 'user123', + userIdsShareesAdded: ['user456', 'user789'], + shareesRemoved: null, + }, + }); + }); + + it('should log on `credentials-created` event', () => { + const event: RelayEventMap['credentials-created'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'Test', + lastName: 'User', + role: 'global:owner', + }, + credentialType: 'githubApi', + credentialId: 'cred456', + publicApi: false, + projectId: 'proj789', + projectType: 'Personal', + }; + + eventService.emit('credentials-created', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.credentials.created', + payload: { + userId: 'user123', + _email: 'user@example.com', + _firstName: 'Test', + _lastName: 'User', + globalRole: 'global:owner', + credentialType: 'githubApi', + credentialId: 'cred456', + publicApi: false, + projectId: 'proj789', + projectType: 'Personal', + }, + }); + }); + + it('should log on `credentials-deleted` event', () => { + const event: RelayEventMap['credentials-deleted'] = { + user: { + id: 'user707', + email: 'creduser@example.com', + firstName: 'Cred', + lastName: 'User', + role: 'global:owner', + }, + credentialId: 'cred789', + credentialType: 'githubApi', + }; + + eventService.emit('credentials-deleted', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.credentials.deleted', + payload: { + userId: 'user707', + _email: 'creduser@example.com', + _firstName: 'Cred', + _lastName: 'User', + globalRole: 'global:owner', + credentialId: 'cred789', + credentialType: 'githubApi', + }, + }); + }); + + it('should log on `credentials-updated` event', () => { + const event: RelayEventMap['credentials-updated'] = { + user: { + id: 'user808', + email: 'updatecred@example.com', + firstName: 'Update', + lastName: 'Cred', + role: 'global:owner', + }, + credentialId: 'cred101', + credentialType: 'slackApi', + }; + + eventService.emit('credentials-updated', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.credentials.updated', + payload: { + userId: 'user808', + _email: 'updatecred@example.com', + _firstName: 'Update', + _lastName: 'Cred', + globalRole: 'global:owner', + credentialId: 'cred101', + credentialType: 'slackApi', + }, + }); + }); + }); + + describe('auth events', () => { + it('should log on `user-login-failed` event', () => { + const event: RelayEventMap['user-login-failed'] = { + userEmail: 'user@example.com', + authenticationMethod: 'email', + reason: 'Invalid password', + }; + + eventService.emit('user-login-failed', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.login.failed', + payload: { + userEmail: 'user@example.com', + authenticationMethod: 'email', + reason: 'Invalid password', + }, + }); + }); + }); + + describe('community package events', () => { + it('should log on `community-package-updated` event', () => { + const event: RelayEventMap['community-package-updated'] = { + user: { + id: 'user202', + email: 'packageupdater@example.com', + firstName: 'Package', + lastName: 'Updater', + role: 'global:admin', + }, + packageName: 'n8n-nodes-awesome-package', + packageVersionCurrent: '1.0.0', + packageVersionNew: '1.1.0', + packageNodeNames: ['AwesomeNode1', 'AwesomeNode2'], + packageAuthor: 'Jane Doe', + packageAuthorEmail: 'jane@example.com', + }; + + eventService.emit('community-package-updated', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.package.updated', + payload: { + userId: 'user202', + _email: 'packageupdater@example.com', + _firstName: 'Package', + _lastName: 'Updater', + globalRole: 'global:admin', + packageName: 'n8n-nodes-awesome-package', + packageVersionCurrent: '1.0.0', + packageVersionNew: '1.1.0', + packageNodeNames: ['AwesomeNode1', 'AwesomeNode2'], + packageAuthor: 'Jane Doe', + packageAuthorEmail: 'jane@example.com', + }, + }); + }); + + it('should log on `community-package-installed` event', () => { + const event: RelayEventMap['community-package-installed'] = { + user: { + id: 'user789', + email: 'admin@example.com', + firstName: 'Admin', + lastName: 'User', + role: 'global:admin', + }, + inputString: 'n8n-nodes-custom-package', + packageName: 'n8n-nodes-custom-package', + success: true, + packageVersion: '1.0.0', + packageNodeNames: ['CustomNode1', 'CustomNode2'], + packageAuthor: 'John Doe', + packageAuthorEmail: 'john@example.com', + }; + + eventService.emit('community-package-installed', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.package.installed', + payload: { + userId: 'user789', + _email: 'admin@example.com', + _firstName: 'Admin', + _lastName: 'User', + globalRole: 'global:admin', + inputString: 'n8n-nodes-custom-package', + packageName: 'n8n-nodes-custom-package', + success: true, + packageVersion: '1.0.0', + packageNodeNames: ['CustomNode1', 'CustomNode2'], + packageAuthor: 'John Doe', + packageAuthorEmail: 'john@example.com', + }, + }); + }); + + it('should log on `community-package-deleted` event', () => { + const event: RelayEventMap['community-package-deleted'] = { + user: { + id: 'user909', + email: 'packagedeleter@example.com', + firstName: 'Package', + lastName: 'Deleter', + role: 'global:admin', + }, + packageName: 'n8n-nodes-awesome-package', + packageVersion: '1.0.0', + packageNodeNames: ['AwesomeNode1', 'AwesomeNode2'], + packageAuthor: 'John Doe', + packageAuthorEmail: 'john@example.com', + }; + + eventService.emit('community-package-deleted', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.package.deleted', + payload: { + userId: 'user909', + _email: 'packagedeleter@example.com', + _firstName: 'Package', + _lastName: 'Deleter', + globalRole: 'global:admin', + packageName: 'n8n-nodes-awesome-package', + packageVersion: '1.0.0', + packageNodeNames: ['AwesomeNode1', 'AwesomeNode2'], + packageAuthor: 'John Doe', + packageAuthorEmail: 'john@example.com', + }, + }); + }); + }); + + describe('email events', () => { + it('should log on `email-failed` event', () => { + const event: RelayEventMap['email-failed'] = { + user: { + id: 'user789', + email: 'recipient@example.com', + firstName: 'Failed', + lastName: 'Recipient', + role: 'global:member', + }, + messageType: 'New user invite', + publicApi: false, + }; + + eventService.emit('email-failed', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.email.failed', + payload: { + userId: 'user789', + _email: 'recipient@example.com', + _firstName: 'Failed', + _lastName: 'Recipient', + globalRole: 'global:member', + messageType: 'New user invite', + }, + }); + }); + }); + + describe('public API events', () => { + it('should log on `public-api-key-created` event', () => { + const event: RelayEventMap['public-api-key-created'] = { + user: { + id: 'user101', + email: 'apiuser@example.com', + firstName: 'API', + lastName: 'User', + role: 'global:owner', + }, + publicApi: true, + }; + + eventService.emit('public-api-key-created', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.api.created', + payload: { + userId: 'user101', + _email: 'apiuser@example.com', + _firstName: 'API', + _lastName: 'User', + globalRole: 'global:owner', + }, + }); + }); + + it('should log on `public-api-key-deleted` event', () => { + const event: RelayEventMap['public-api-key-deleted'] = { + user: { + id: 'user606', + email: 'apiuser@example.com', + firstName: 'API', + lastName: 'User', + role: 'global:owner', + }, + publicApi: true, + }; + + eventService.emit('public-api-key-deleted', event); + + expect(eventBus.sendAuditEvent).toHaveBeenCalledWith({ + eventName: 'n8n.audit.user.api.deleted', + payload: { + userId: 'user606', + _email: 'apiuser@example.com', + _firstName: 'API', + _lastName: 'User', + globalRole: 'global:owner', + }, + }); + }); + }); + + describe('execution events', () => { + it('should log on `execution-started-during-bootup` event', () => { + const event: RelayEventMap['execution-started-during-bootup'] = { + executionId: 'exec101010', + }; + + eventService.emit('execution-started-during-bootup', event); + + expect(eventBus.sendExecutionEvent).toHaveBeenCalledWith({ + eventName: 'n8n.execution.started-during-bootup', + payload: { + executionId: 'exec101010', + }, + }); + }); + + it('should log on `execution-throttled` event', () => { + const event: RelayEventMap['execution-throttled'] = { + executionId: 'exec123456', + }; + + eventService.emit('execution-throttled', event); + + expect(eventBus.sendExecutionEvent).toHaveBeenCalledWith({ + eventName: 'n8n.execution.throttled', + payload: { + executionId: 'exec123456', + }, + }); + }); + }); +}); diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts new file mode 100644 index 0000000000..87d89841ad --- /dev/null +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -0,0 +1,1053 @@ +import { mock } from 'jest-mock-extended'; +import { TelemetryEventRelay } from '@/events/telemetry-event-relay'; +import { EventService } from '@/events/event.service'; +import config from '@/config'; +import type { IWorkflowBase } from 'n8n-workflow'; +import type { IWorkflowDb } from '@/Interfaces'; +import type { Telemetry } from '@/telemetry'; +import type { License } from '@/license'; +import type { GlobalConfig } from '@n8n/config'; +import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import type { NodeTypes } from '@/node-types'; +import type { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflow.repository'; +import type { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; +import type { RelayEventMap } from '@/events/relay-event-map'; +import type { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; +import { N8N_VERSION } from '@/constants'; + +const flushPromises = async () => await new Promise((resolve) => setImmediate(resolve)); + +describe('TelemetryEventRelay', () => { + const telemetry = mock(); + const license = mock(); + const globalConfig = mock({ + userManagement: { + emails: { + mode: 'smtp', + }, + }, + endpoints: { + metrics: { + enable: true, + includeDefaultMetrics: true, + includeApiEndpoints: false, + includeCacheMetrics: false, + includeMessageEventBusMetrics: false, + }, + }, + }); + const workflowRepository = mock(); + const nodeTypes = mock(); + const sharedWorkflowRepository = mock(); + const projectRelationRepository = mock(); + const eventService = new EventService(); + + let telemetryEventRelay: TelemetryEventRelay; + + beforeAll(async () => { + telemetryEventRelay = new TelemetryEventRelay( + eventService, + telemetry, + license, + globalConfig, + workflowRepository, + nodeTypes, + sharedWorkflowRepository, + projectRelationRepository, + ); + + await telemetryEventRelay.init(); + }); + + beforeEach(() => { + config.set('diagnostics.enabled', true); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('init', () => { + it('with diagnostics enabled, should init telemetry and register listeners', async () => { + config.set('diagnostics.enabled', true); + const telemetryEventRelay = new TelemetryEventRelay( + eventService, + telemetry, + license, + globalConfig, + workflowRepository, + nodeTypes, + sharedWorkflowRepository, + projectRelationRepository, + ); + // @ts-expect-error Private method + const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners'); + + await telemetryEventRelay.init(); + + expect(telemetry.init).toHaveBeenCalled(); + expect(setupListenersSpy).toHaveBeenCalled(); + }); + + it('with diagnostics disabled, should neither init telemetry nor register listeners', async () => { + config.set('diagnostics.enabled', false); + const telemetryEventRelay = new TelemetryEventRelay( + eventService, + telemetry, + license, + globalConfig, + workflowRepository, + nodeTypes, + sharedWorkflowRepository, + projectRelationRepository, + ); + // @ts-expect-error Private method + const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners'); + + await telemetryEventRelay.init(); + + expect(telemetry.init).not.toHaveBeenCalled(); + expect(setupListenersSpy).not.toHaveBeenCalled(); + }); + }); + + describe('project events', () => { + it('should track on `team-project-updated` event', () => { + const event: RelayEventMap['team-project-updated'] = { + userId: 'user123', + role: 'global:owner', + members: [ + { userId: 'user456', role: 'project:admin' }, + { userId: 'user789', role: 'project:editor' }, + ], + projectId: 'project123', + }; + + eventService.emit('team-project-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('Project settings updated', { + user_id: 'user123', + role: 'global:owner', + members: [ + { user_id: 'user456', role: 'project:admin' }, + { user_id: 'user789', role: 'project:editor' }, + ], + project_id: 'project123', + }); + }); + + it('should track on `team-project-deleted` event', () => { + const event: RelayEventMap['team-project-deleted'] = { + userId: 'user123', + role: 'global:owner', + projectId: 'project123', + removalType: 'delete', + }; + + eventService.emit('team-project-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('User deleted project', { + user_id: 'user123', + role: 'global:owner', + project_id: 'project123', + removal_type: 'delete', + target_project_id: undefined, + }); + }); + + it('should track on `team-project-created` event', () => { + const event: RelayEventMap['team-project-created'] = { + userId: 'user123', + role: 'global:owner', + }; + + eventService.emit('team-project-created', event); + + expect(telemetry.track).toHaveBeenCalledWith('User created project', { + user_id: 'user123', + role: 'global:owner', + }); + }); + }); + + describe('source control events', () => { + it('should track on `source-control-settings-updated` event', () => { + const event: RelayEventMap['source-control-settings-updated'] = { + branchName: 'main', + readOnlyInstance: false, + repoType: 'github', + connected: true, + }; + + eventService.emit('source-control-settings-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('User updated source control settings', { + branch_name: 'main', + read_only_instance: false, + repo_type: 'github', + connected: true, + }); + }); + + it('should track on `source-control-user-started-pull-ui` event', () => { + const event: RelayEventMap['source-control-user-started-pull-ui'] = { + workflowUpdates: 5, + workflowConflicts: 2, + credConflicts: 1, + }; + + eventService.emit('source-control-user-started-pull-ui', event); + + expect(telemetry.track).toHaveBeenCalledWith('User started pull via UI', { + workflow_updates: 5, + workflow_conflicts: 2, + cred_conflicts: 1, + }); + }); + + it('should track on `source-control-user-finished-pull-ui` event', () => { + const event: RelayEventMap['source-control-user-finished-pull-ui'] = { + workflowUpdates: 3, + }; + + eventService.emit('source-control-user-finished-pull-ui', event); + + expect(telemetry.track).toHaveBeenCalledWith('User finished pull via UI', { + workflow_updates: 3, + }); + }); + + it('should track on `source-control-user-pulled-api` event', () => { + const event: RelayEventMap['source-control-user-pulled-api'] = { + workflowUpdates: 2, + forced: false, + }; + + eventService.emit('source-control-user-pulled-api', event); + + expect(telemetry.track).toHaveBeenCalledWith('User pulled via API', { + workflow_updates: 2, + forced: false, + }); + }); + + it('should track on `source-control-user-started-push-ui` event', () => { + const event: RelayEventMap['source-control-user-started-push-ui'] = { + workflowsEligible: 10, + workflowsEligibleWithConflicts: 2, + credsEligible: 5, + credsEligibleWithConflicts: 1, + variablesEligible: 3, + }; + + eventService.emit('source-control-user-started-push-ui', event); + + expect(telemetry.track).toHaveBeenCalledWith('User started push via UI', { + workflows_eligible: 10, + workflows_eligible_with_conflicts: 2, + creds_eligible: 5, + creds_eligible_with_conflicts: 1, + variables_eligible: 3, + }); + }); + + it('should track on `source-control-user-finished-push-ui` event', () => { + const event: RelayEventMap['source-control-user-finished-push-ui'] = { + workflowsEligible: 10, + workflowsPushed: 8, + credsPushed: 5, + variablesPushed: 3, + }; + + eventService.emit('source-control-user-finished-push-ui', event); + + expect(telemetry.track).toHaveBeenCalledWith('User finished push via UI', { + workflows_eligible: 10, + workflows_pushed: 8, + creds_pushed: 5, + variables_pushed: 3, + }); + }); + }); + + describe('license events', () => { + it('should track on `license-renewal-attempted` event', () => { + const event: RelayEventMap['license-renewal-attempted'] = { + success: true, + }; + + eventService.emit('license-renewal-attempted', event); + + expect(telemetry.track).toHaveBeenCalledWith('Instance attempted to refresh license', { + success: true, + }); + }); + }); + + describe('variable events', () => { + it('should track on `variable-created` event', () => { + eventService.emit('variable-created', {}); + + expect(telemetry.track).toHaveBeenCalledWith('User created variable'); + }); + }); + + describe('external secrets events', () => { + it('should track on `external-secrets-provider-settings-saved` event', () => { + const event: RelayEventMap['external-secrets-provider-settings-saved'] = { + userId: 'user123', + vaultType: 'aws', + isValid: true, + isNew: false, + }; + + eventService.emit('external-secrets-provider-settings-saved', event); + + expect(telemetry.track).toHaveBeenCalledWith('User updated external secrets settings', { + user_id: 'user123', + vault_type: 'aws', + is_valid: true, + is_new: false, + error_message: undefined, + }); + }); + }); + + describe('public API events', () => { + it('should track on `public-api-invoked` event', () => { + const event: RelayEventMap['public-api-invoked'] = { + userId: 'user123', + path: '/api/v1/workflows', + method: 'GET', + apiVersion: 'v1', + }; + + eventService.emit('public-api-invoked', event); + + expect(telemetry.track).toHaveBeenCalledWith('User invoked API', { + user_id: 'user123', + path: '/api/v1/workflows', + method: 'GET', + api_version: 'v1', + }); + }); + + it('should track on `public-api-key-created` event', () => { + const event: RelayEventMap['public-api-key-created'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + publicApi: true, + }; + + eventService.emit('public-api-key-created', event); + + expect(telemetry.track).toHaveBeenCalledWith('API key created', { + user_id: 'user123', + public_api: true, + }); + }); + + it('should track on `public-api-key-deleted` event', () => { + const event: RelayEventMap['public-api-key-deleted'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + publicApi: true, + }; + + eventService.emit('public-api-key-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('API key deleted', { + user_id: 'user123', + public_api: true, + }); + }); + }); + + describe('community package events', () => { + it('should track on `community-package-installed` event', () => { + const event: RelayEventMap['community-package-installed'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + inputString: 'n8n-nodes-package', + packageName: 'n8n-nodes-package', + success: true, + packageVersion: '1.0.0', + packageNodeNames: ['CustomNode1', 'CustomNode2'], + packageAuthor: 'John Smith', + packageAuthorEmail: 'john@example.com', + }; + + eventService.emit('community-package-installed', event); + + expect(telemetry.track).toHaveBeenCalledWith('cnr package install finished', { + user_id: 'user123', + input_string: 'n8n-nodes-package', + package_name: 'n8n-nodes-package', + success: true, + package_version: '1.0.0', + package_node_names: ['CustomNode1', 'CustomNode2'], + package_author: 'John Smith', + package_author_email: 'john@example.com', + failure_reason: undefined, + }); + }); + + it('should track on `community-package-updated` event', () => { + const event: RelayEventMap['community-package-updated'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + packageName: 'n8n-nodes-package', + packageVersionCurrent: '1.0.0', + packageVersionNew: '1.1.0', + packageNodeNames: ['CustomNode1', 'CustomNode2'], + packageAuthor: 'John Smith', + packageAuthorEmail: 'john@example.com', + }; + + eventService.emit('community-package-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('cnr package updated', { + user_id: 'user123', + package_name: 'n8n-nodes-package', + package_version_current: '1.0.0', + package_version_new: '1.1.0', + package_node_names: ['CustomNode1', 'CustomNode2'], + package_author: 'John Smith', + package_author_email: 'john@example.com', + }); + }); + + it('should track on `community-package-deleted` event', () => { + const event: RelayEventMap['community-package-deleted'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + packageName: 'n8n-nodes-package', + packageVersion: '1.0.0', + packageNodeNames: ['CustomNode1', 'CustomNode2'], + packageAuthor: 'John Smith', + packageAuthorEmail: 'john@example.com', + }; + + eventService.emit('community-package-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('cnr package deleted', { + user_id: 'user123', + package_name: 'n8n-nodes-package', + package_version: '1.0.0', + package_node_names: ['CustomNode1', 'CustomNode2'], + package_author: 'John Smith', + package_author_email: 'john@example.com', + }); + }); + }); + + describe('credentials events', () => { + it('should track on `credentials-created` event', () => { + const event: RelayEventMap['credentials-created'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + credentialType: 'github', + credentialId: 'cred123', + publicApi: false, + projectId: 'project123', + projectType: 'personal', + }; + + eventService.emit('credentials-created', event); + + expect(telemetry.track).toHaveBeenCalledWith('User created credentials', { + user_id: 'user123', + credential_type: 'github', + credential_id: 'cred123', + project_id: 'project123', + project_type: 'personal', + }); + }); + + it('should track on `credentials-shared` event', () => { + const event: RelayEventMap['credentials-shared'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + credentialType: 'github', + credentialId: 'cred123', + userIdSharer: 'user123', + userIdsShareesAdded: ['user456', 'user789'], + shareesRemoved: 1, + }; + + eventService.emit('credentials-shared', event); + + expect(telemetry.track).toHaveBeenCalledWith('User updated cred sharing', { + user_id: 'user123', + credential_type: 'github', + credential_id: 'cred123', + user_id_sharer: 'user123', + user_ids_sharees_added: ['user456', 'user789'], + sharees_removed: 1, + }); + }); + + it('should track on `credentials-updated` event', () => { + const event: RelayEventMap['credentials-updated'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + credentialId: 'cred123', + credentialType: 'github', + }; + + eventService.emit('credentials-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('User updated credentials', { + user_id: 'user123', + credential_type: 'github', + credential_id: 'cred123', + }); + }); + + it('should track on `credentials-deleted` event', () => { + const event: RelayEventMap['credentials-deleted'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + credentialId: 'cred123', + credentialType: 'github', + }; + + eventService.emit('credentials-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('User deleted credentials', { + user_id: 'user123', + credential_type: 'github', + credential_id: 'cred123', + }); + }); + }); + + describe('LDAP events', () => { + it('should track on `ldap-general-sync-finished` event', () => { + const event: RelayEventMap['ldap-general-sync-finished'] = { + type: 'full', + succeeded: true, + usersSynced: 10, + error: '', + }; + + eventService.emit('ldap-general-sync-finished', event); + + expect(telemetry.track).toHaveBeenCalledWith('Ldap general sync finished', { + type: 'full', + succeeded: true, + users_synced: 10, + error: '', + }); + }); + + it('should track on `ldap-settings-updated` event', () => { + const event: RelayEventMap['ldap-settings-updated'] = { + userId: 'user123', + loginIdAttribute: 'uid', + firstNameAttribute: 'givenName', + lastNameAttribute: 'sn', + emailAttribute: 'mail', + ldapIdAttribute: 'entryUUID', + searchPageSize: 100, + searchTimeout: 60, + synchronizationEnabled: true, + synchronizationInterval: 60, + loginLabel: 'LDAP Login', + loginEnabled: true, + }; + + eventService.emit('ldap-settings-updated', { + ...event, + }); + + const { userId: _, ...rest } = event; + + expect(telemetry.track).toHaveBeenCalledWith('User updated Ldap settings', { + user_id: 'user123', + ...rest, + }); + }); + + it('should track on `ldap-login-sync-failed` event', () => { + const event: RelayEventMap['ldap-login-sync-failed'] = { + error: 'Connection failed', + }; + + eventService.emit('ldap-login-sync-failed', event); + + expect(telemetry.track).toHaveBeenCalledWith('Ldap login sync failed', { + error: 'Connection failed', + }); + }); + + it('should track on `login-failed-due-to-ldap-disabled` event', () => { + const event: RelayEventMap['login-failed-due-to-ldap-disabled'] = { + userId: 'user123', + }; + + eventService.emit('login-failed-due-to-ldap-disabled', event); + + expect(telemetry.track).toHaveBeenCalledWith('User login failed since ldap disabled', { + user_ud: 'user123', + }); + }); + }); + + describe('workflow events', () => { + it('should track on `workflow-created` event', async () => { + const event: RelayEventMap['workflow-created'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + workflow: mock({ id: 'workflow123', name: 'Test Workflow', nodes: [] }), + publicApi: false, + projectId: 'project123', + projectType: 'personal', + }; + + eventService.emit('workflow-created', event); + + await flushPromises(); + + expect(telemetry.track).toHaveBeenCalledWith('User created workflow', { + user_id: 'user123', + workflow_id: 'workflow123', + node_graph_string: expect.any(String), + public_api: false, + project_id: 'project123', + project_type: 'personal', + }); + }); + + it('should track on `workflow-deleted` event', () => { + const event: RelayEventMap['workflow-deleted'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + workflowId: 'workflow123', + publicApi: false, + }; + + eventService.emit('workflow-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('User deleted workflow', { + user_id: 'user123', + workflow_id: 'workflow123', + public_api: false, + }); + }); + + it('should track on `workflow-post-execute` event', async () => { + const event: RelayEventMap['workflow-post-execute'] = { + workflow: mock({ + id: 'workflow123', + name: 'Test Workflow', + nodes: [], + }), + userId: 'user123', + executionId: 'execution123', + }; + + eventService.emit('workflow-post-execute', event); + + await flushPromises(); + + expect(telemetry.trackWorkflowExecution).toHaveBeenCalledWith({ + is_manual: false, + success: false, + user_id: 'user123', + version_cli: N8N_VERSION, + workflow_id: 'workflow123', + }); + }); + + it('should track on `workflow-saved` event', async () => { + const event: RelayEventMap['workflow-saved'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + workflow: mock({ id: 'workflow123', name: 'Test Workflow', nodes: [] }), + publicApi: false, + }; + + eventService.emit('workflow-saved', event); + + await flushPromises(); + + expect(telemetry.track).toHaveBeenCalledWith('User saved workflow', { + user_id: 'user123', + workflow_id: 'workflow123', + node_graph_string: expect.any(String), + notes_count_overlapping: 0, + notes_count_non_overlapping: 0, + version_cli: expect.any(String), + num_tags: 0, + public_api: false, + sharing_role: undefined, + }); + }); + + it('should track on `workflow-sharing-updated` event', () => { + const event: RelayEventMap['workflow-sharing-updated'] = { + workflowId: 'workflow123', + userIdSharer: 'user123', + userIdList: ['user456', 'user789'], + }; + + eventService.emit('workflow-sharing-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('User updated workflow sharing', { + workflow_id: 'workflow123', + user_id_sharer: 'user123', + user_id_list: ['user456', 'user789'], + }); + }); + }); + + describe('user events', () => { + it('should track on `user-updated` event', () => { + const event: RelayEventMap['user-updated'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + fieldsChanged: ['firstName', 'lastName'], + }; + + eventService.emit('user-updated', event); + + expect(telemetry.track).toHaveBeenCalledWith('User changed personal settings', { + user_id: 'user123', + fields_changed: ['firstName', 'lastName'], + }); + }); + + it('should track on `user-deleted` event', () => { + const event: RelayEventMap['user-deleted'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + publicApi: false, + targetUserOldStatus: 'active', + migrationStrategy: 'transfer_data', + targetUserId: 'user456', + migrationUserId: 'user789', + }; + + eventService.emit('user-deleted', event); + + expect(telemetry.track).toHaveBeenCalledWith('User deleted user', { + user_id: 'user123', + public_api: false, + target_user_old_status: 'active', + migration_strategy: 'transfer_data', + target_user_id: 'user456', + migration_user_id: 'user789', + }); + }); + + it('should track on `user-invited` event', () => { + const event: RelayEventMap['user-invited'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + targetUserId: ['user456'], + publicApi: false, + emailSent: true, + inviteeRole: 'global:member', + }; + + eventService.emit('user-invited', event); + + expect(telemetry.track).toHaveBeenCalledWith('User invited new user', { + user_id: 'user123', + target_user_id: ['user456'], + public_api: false, + email_sent: true, + invitee_role: 'global:member', + }); + }); + + it('should track on `user-signed-up` event', () => { + const event: RelayEventMap['user-signed-up'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + userType: 'email', + wasDisabledLdapUser: false, + }; + + eventService.emit('user-signed-up', event); + + expect(telemetry.track).toHaveBeenCalledWith('User signed up', { + user_id: 'user123', + user_type: 'email', + was_disabled_ldap_user: false, + }); + }); + + it('should track on `user-submitted-personalization-survey` event', () => { + const event: RelayEventMap['user-submitted-personalization-survey'] = { + userId: 'user123', + answers: { + version: 'v4', + personalization_survey_n8n_version: '1.0.0', + personalization_survey_submitted_at: '2021-10-01T00:00:00.000Z', + companySize: '1-10', + }, + }; + + eventService.emit('user-submitted-personalization-survey', event); + + expect(telemetry.track).toHaveBeenCalledWith('User responded to personalization questions', { + user_id: 'user123', + version: 'v4', + personalization_survey_n8n_version: '1.0.0', + personalization_survey_submitted_at: '2021-10-01T00:00:00.000Z', + company_size: '1-10', + }); + }); + + it('should track on `user-changed-role` event', () => { + const event: RelayEventMap['user-changed-role'] = { + userId: 'user123', + targetUserId: 'user456', + targetUserNewRole: 'global:member', + publicApi: false, + }; + + eventService.emit('user-changed-role', event); + + expect(telemetry.track).toHaveBeenCalledWith('User changed role', { + user_id: 'user123', + target_user_id: 'user456', + target_user_new_role: 'global:member', + public_api: false, + }); + }); + + it('should track on `user-retrieved-user` event', () => { + const event: RelayEventMap['user-retrieved-user'] = { + userId: 'user123', + publicApi: false, + }; + + eventService.emit('user-retrieved-user', event); + + expect(telemetry.track).toHaveBeenCalledWith('User retrieved user', { + user_id: 'user123', + public_api: false, + }); + }); + + it('should track on `user-retrieved-all-users` event', () => { + const event: RelayEventMap['user-retrieved-all-users'] = { + userId: 'user123', + publicApi: false, + }; + + eventService.emit('user-retrieved-all-users', event); + + expect(telemetry.track).toHaveBeenCalledWith('User retrieved all users', { + user_id: 'user123', + public_api: false, + }); + }); + }); + + describe('lifecycle events', () => { + it('should track on `server-started` event', async () => { + const firstWorkflow = mock({ createdAt: new Date() }); + workflowRepository.findOne.mockResolvedValue(firstWorkflow); + + eventService.emit('server-started'); + + await flushPromises(); + + // expect(telemetry.identify).toHaveBeenCalled(); + expect(telemetry.track).toHaveBeenCalledWith( + 'Instance started', + expect.objectContaining({ + earliest_workflow_created: firstWorkflow.createdAt, + metrics: { + metrics_enabled: true, + metrics_category_default: true, + metrics_category_routes: false, + metrics_category_cache: false, + metrics_category_logs: false, + }, + }), + ); + }); + + it('should track on `session-started` event', () => { + const event: RelayEventMap['session-started'] = { + pushRef: 'ref123', + }; + + eventService.emit('session-started', event); + + expect(telemetry.track).toHaveBeenCalledWith('Session started', { + session_id: 'ref123', + }); + }); + + it('should track on `instance-stopped` event', () => { + eventService.emit('instance-stopped', {}); + + expect(telemetry.track).toHaveBeenCalledWith('User instance stopped'); + }); + + it('should track on `instance-owner-setup` event', () => { + const event: RelayEventMap['instance-owner-setup'] = { + userId: 'user123', + }; + + eventService.emit('instance-owner-setup', event); + + expect(telemetry.track).toHaveBeenCalledWith('Owner finished instance setup', { + user_id: 'user123', + }); + }); + }); + + describe('workflow execution events', () => { + it('should track on `first-production-workflow-succeeded` event', () => { + const event: RelayEventMap['first-production-workflow-succeeded'] = { + projectId: 'project123', + workflowId: 'workflow123', + userId: 'user123', + }; + + eventService.emit('first-production-workflow-succeeded', event); + + expect(telemetry.track).toHaveBeenCalledWith('Workflow first prod success', { + project_id: 'project123', + workflow_id: 'workflow123', + user_id: 'user123', + }); + }); + + it('should track on `first-workflow-data-loaded` event', () => { + const event: RelayEventMap['first-workflow-data-loaded'] = { + userId: 'user123', + workflowId: 'workflow123', + nodeType: 'http', + nodeId: 'node123', + credentialType: 'oAuth2', + credentialId: 'cred123', + }; + + eventService.emit('first-workflow-data-loaded', event); + + expect(telemetry.track).toHaveBeenCalledWith('Workflow first data fetched', { + user_id: 'user123', + workflow_id: 'workflow123', + node_type: 'http', + node_id: 'node123', + credential_type: 'oAuth2', + credential_id: 'cred123', + }); + }); + }); + + describe('email events', () => { + it('should track on `email-failed` event', () => { + const event: RelayEventMap['email-failed'] = { + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: 'global:owner', + }, + messageType: 'New user invite', + publicApi: false, + }; + + eventService.emit('email-failed', event); + + expect(telemetry.track).toHaveBeenCalledWith( + 'Instance failed to send transactional email to user', + { + user_id: 'user123', + message_type: 'New user invite', + public_api: false, + }, + ); + }); + }); +}); diff --git a/packages/cli/src/events/event-relay.ts b/packages/cli/src/events/event-relay.ts new file mode 100644 index 0000000000..1a8a17b893 --- /dev/null +++ b/packages/cli/src/events/event-relay.ts @@ -0,0 +1,20 @@ +import { EventService } from './event.service'; +import { Service } from 'typedi'; +import type { RelayEventMap } from '@/events/relay-event-map'; + +@Service() +export class EventRelay { + constructor(readonly eventService: EventService) {} + + protected setupListeners(map: { + [EventName in EventNames]?: (event: RelayEventMap[EventName]) => void | Promise; + }) { + for (const [eventName, handler] of Object.entries(map) as Array< + [EventNames, (event: RelayEventMap[EventNames]) => void | Promise] + >) { + this.eventService.on(eventName, async (event) => { + await handler(event); + }); + } + } +} diff --git a/packages/cli/src/events/event.service.ts b/packages/cli/src/events/event.service.ts new file mode 100644 index 0000000000..6744103a07 --- /dev/null +++ b/packages/cli/src/events/event.service.ts @@ -0,0 +1,6 @@ +import { Service } from 'typedi'; +import { TypedEmitter } from '@/TypedEmitter'; +import type { RelayEventMap } from './relay-event-map'; + +@Service() +export class EventService extends TypedEmitter {} diff --git a/packages/cli/src/events/log-streaming-event-relay.ts b/packages/cli/src/events/log-streaming-event-relay.ts new file mode 100644 index 0000000000..2012775d91 --- /dev/null +++ b/packages/cli/src/events/log-streaming-event-relay.ts @@ -0,0 +1,390 @@ +import { Service } from 'typedi'; +import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; +import { Redactable } from '@/decorators/redactable'; +import { EventRelay } from '@/events/event-relay'; +import type { RelayEventMap } from '@/events/relay-event-map'; +import type { IWorkflowBase } from 'n8n-workflow'; +import { EventService } from './event.service'; + +@Service() +export class LogStreamingEventRelay extends EventRelay { + constructor( + readonly eventService: EventService, + private readonly eventBus: MessageEventBus, + ) { + super(eventService); + } + + init() { + this.setupListeners({ + 'workflow-created': (event) => this.workflowCreated(event), + 'workflow-deleted': (event) => this.workflowDeleted(event), + 'workflow-saved': (event) => this.workflowSaved(event), + 'workflow-pre-execute': (event) => this.workflowPreExecute(event), + 'workflow-post-execute': (event) => this.workflowPostExecute(event), + 'node-pre-execute': (event) => this.nodePreExecute(event), + 'node-post-execute': (event) => this.nodePostExecute(event), + 'user-deleted': (event) => this.userDeleted(event), + 'user-invited': (event) => this.userInvited(event), + 'user-reinvited': (event) => this.userReinvited(event), + 'user-updated': (event) => this.userUpdated(event), + 'user-signed-up': (event) => this.userSignedUp(event), + 'user-logged-in': (event) => this.userLoggedIn(event), + 'user-login-failed': (event) => this.userLoginFailed(event), + 'user-invite-email-click': (event) => this.userInviteEmailClick(event), + 'user-password-reset-email-click': (event) => this.userPasswordResetEmailClick(event), + 'user-password-reset-request-click': (event) => this.userPasswordResetRequestClick(event), + 'public-api-key-created': (event) => this.publicApiKeyCreated(event), + 'public-api-key-deleted': (event) => this.publicApiKeyDeleted(event), + 'email-failed': (event) => this.emailFailed(event), + 'credentials-created': (event) => this.credentialsCreated(event), + 'credentials-deleted': (event) => this.credentialsDeleted(event), + 'credentials-shared': (event) => this.credentialsShared(event), + 'credentials-updated': (event) => this.credentialsUpdated(event), + 'community-package-installed': (event) => this.communityPackageInstalled(event), + 'community-package-updated': (event) => this.communityPackageUpdated(event), + 'community-package-deleted': (event) => this.communityPackageDeleted(event), + 'execution-throttled': (event) => this.executionThrottled(event), + 'execution-started-during-bootup': (event) => this.executionStartedDuringBootup(event), + }); + } + + // #region Workflow + + @Redactable() + private workflowCreated({ user, workflow }: RelayEventMap['workflow-created']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.created', + payload: { + ...user, + workflowId: workflow.id, + workflowName: workflow.name, + }, + }); + } + + @Redactable() + private workflowDeleted({ user, workflowId }: RelayEventMap['workflow-deleted']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.deleted', + payload: { ...user, workflowId }, + }); + } + + @Redactable() + private workflowSaved({ user, workflow }: RelayEventMap['workflow-saved']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.updated', + payload: { + ...user, + workflowId: workflow.id, + workflowName: workflow.name, + }, + }); + } + + private workflowPreExecute({ data, executionId }: RelayEventMap['workflow-pre-execute']) { + const payload = + 'executionData' in data + ? { + executionId, + userId: data.userId, + workflowId: data.workflowData.id, + isManual: data.executionMode === 'manual', + workflowName: data.workflowData.name, + } + : { + executionId, + userId: undefined, + workflowId: (data as IWorkflowBase).id, + isManual: false, + workflowName: (data as IWorkflowBase).name, + }; + + void this.eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.started', + payload, + }); + } + + private workflowPostExecute(event: RelayEventMap['workflow-post-execute']) { + const { runData, workflow, ...rest } = event; + + const payload = { + ...rest, + success: !!runData?.finished, // despite the `success` name, this reports `finished` state + isManual: runData?.mode === 'manual', + workflowId: workflow.id, + workflowName: workflow.name, + }; + + if (payload.success) { + void this.eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.success', + payload, + }); + + return; + } + + void this.eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.failed', + payload: { + ...payload, + lastNodeExecuted: runData?.data.resultData.lastNodeExecuted, + errorNodeType: + runData?.data.resultData.error && 'node' in runData?.data.resultData.error + ? runData?.data.resultData.error.node?.type + : undefined, + errorMessage: runData?.data.resultData.error?.message.toString(), + }, + }); + } + + // #endregion + + // #region Node + + private nodePreExecute({ workflow, executionId, nodeName }: RelayEventMap['node-pre-execute']) { + void this.eventBus.sendNodeEvent({ + eventName: 'n8n.node.started', + payload: { + workflowId: workflow.id, + workflowName: workflow.name, + executionId, + nodeType: workflow.nodes.find((n) => n.name === nodeName)?.type, + nodeName, + }, + }); + } + + private nodePostExecute({ workflow, executionId, nodeName }: RelayEventMap['node-post-execute']) { + void this.eventBus.sendNodeEvent({ + eventName: 'n8n.node.finished', + payload: { + workflowId: workflow.id, + workflowName: workflow.name, + executionId, + nodeType: workflow.nodes.find((n) => n.name === nodeName)?.type, + nodeName, + }, + }); + } + + // #endregion + + // #region User + + @Redactable() + private userDeleted({ user }: RelayEventMap['user-deleted']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.deleted', + payload: user, + }); + } + + @Redactable() + private userInvited({ user, targetUserId }: RelayEventMap['user-invited']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.invited', + payload: { ...user, targetUserId }, + }); + } + + @Redactable() + private userReinvited({ user, targetUserId }: RelayEventMap['user-reinvited']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reinvited', + payload: { ...user, targetUserId }, + }); + } + + @Redactable() + private userUpdated({ user, fieldsChanged }: RelayEventMap['user-updated']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.updated', + payload: { ...user, fieldsChanged }, + }); + } + + // #endregion + + // #region Auth + + @Redactable() + private userSignedUp({ user }: RelayEventMap['user-signed-up']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.signedup', + payload: user, + }); + } + + @Redactable() + private userLoggedIn({ user, authenticationMethod }: RelayEventMap['user-logged-in']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.login.success', + payload: { ...user, authenticationMethod }, + }); + } + + private userLoginFailed( + event: RelayEventMap['user-login-failed'] /* exception: no `UserLike` to redact */, + ) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.login.failed', + payload: event, + }); + } + + // #endregion + + // #region Click + + @Redactable('inviter') + @Redactable('invitee') + private userInviteEmailClick(event: RelayEventMap['user-invite-email-click']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.invitation.accepted', + payload: event, + }); + } + + @Redactable() + private userPasswordResetEmailClick({ user }: RelayEventMap['user-password-reset-email-click']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reset', + payload: user, + }); + } + + @Redactable() + private userPasswordResetRequestClick({ + user, + }: RelayEventMap['user-password-reset-request-click']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reset.requested', + payload: user, + }); + } + + // #endregion + + // #region Public API + + @Redactable() + private publicApiKeyCreated({ user }: RelayEventMap['public-api-key-created']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.api.created', + payload: user, + }); + } + + @Redactable() + private publicApiKeyDeleted({ user }: RelayEventMap['public-api-key-deleted']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.api.deleted', + payload: user, + }); + } + + // #endregion + + // #region Email + + @Redactable() + private emailFailed({ user, messageType }: RelayEventMap['email-failed']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.email.failed', + payload: { ...user, messageType }, + }); + } + + // #endregion + + // #region Credentials + + @Redactable() + private credentialsCreated({ user, ...rest }: RelayEventMap['credentials-created']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.created', + payload: { ...user, ...rest }, + }); + } + + @Redactable() + private credentialsDeleted({ user, ...rest }: RelayEventMap['credentials-deleted']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.deleted', + payload: { ...user, ...rest }, + }); + } + + @Redactable() + private credentialsShared({ user, ...rest }: RelayEventMap['credentials-shared']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.shared', + payload: { ...user, ...rest }, + }); + } + + @Redactable() + private credentialsUpdated({ user, ...rest }: RelayEventMap['credentials-updated']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.updated', + payload: { ...user, ...rest }, + }); + } + + // #endregion + + // #region Community package + + @Redactable() + private communityPackageInstalled({ + user, + ...rest + }: RelayEventMap['community-package-installed']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.installed', + payload: { ...user, ...rest }, + }); + } + + @Redactable() + private communityPackageUpdated({ user, ...rest }: RelayEventMap['community-package-updated']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.updated', + payload: { ...user, ...rest }, + }); + } + + @Redactable() + private communityPackageDeleted({ user, ...rest }: RelayEventMap['community-package-deleted']) { + void this.eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.deleted', + payload: { ...user, ...rest }, + }); + } + + // #endregion + + // #region Execution + + private executionThrottled({ executionId }: RelayEventMap['execution-throttled']) { + void this.eventBus.sendExecutionEvent({ + eventName: 'n8n.execution.throttled', + payload: { executionId }, + }); + } + + private executionStartedDuringBootup({ + executionId, + }: RelayEventMap['execution-started-during-bootup']) { + void this.eventBus.sendExecutionEvent({ + eventName: 'n8n.execution.started-during-bootup', + payload: { executionId }, + }); + } + + // #endregion +} diff --git a/packages/cli/src/eventbus/event.types.ts b/packages/cli/src/events/relay-event-map.ts similarity index 63% rename from packages/cli/src/eventbus/event.types.ts rename to packages/cli/src/events/relay-event-map.ts index c7b80357d1..ffc9c9d716 100644 --- a/packages/cli/src/eventbus/event.types.ts +++ b/packages/cli/src/events/relay-event-map.ts @@ -1,7 +1,13 @@ -import type { AuthenticationMethod, IWorkflowBase } from 'n8n-workflow'; -import type { IWorkflowExecutionDataProcess } from '@/Interfaces'; +import type { + AuthenticationMethod, + IPersonalizationSurveyAnswersV4, + IRun, + IWorkflowBase, +} from 'n8n-workflow'; +import type { IWorkflowDb, IWorkflowExecutionDataProcess } from '@/Interfaces'; import type { ProjectRole } from '@/databases/entities/ProjectRelation'; import type { GlobalRole } from '@/databases/entities/User'; +import type { AuthProviderType } from '@/databases/entities/AuthIdentity'; export type UserLike = { id: string; @@ -11,24 +17,58 @@ export type UserLike = { role: string; }; -/** - * Events sent by `EventService` and forwarded by relays, e.g. `AuditEventRelay` and `TelemetryEventRelay`. - */ -export type Event = { +export type RelayEventMap = { + // #region Lifecycle + + 'server-started': {}; + + 'session-started': { + pushRef?: string; + }; + + 'instance-stopped': {}; + + 'instance-owner-setup': { + userId: string; + }; + + 'first-production-workflow-succeeded': { + projectId: string; + workflowId: string; + userId: string; + }; + + 'first-workflow-data-loaded': { + userId: string; + workflowId: string; + nodeType: string; + nodeId: string; + credentialType?: string; + credentialId?: string; + }; + + // #endregion + + // #region Workflow + 'workflow-created': { user: UserLike; workflow: IWorkflowBase; + publicApi: boolean; + projectId: string; + projectType: string; }; 'workflow-deleted': { user: UserLike; workflowId: string; + publicApi: boolean; }; 'workflow-saved': { user: UserLike; - workflowId: string; - workflowName: string; + workflow: IWorkflowDb; + publicApi: boolean; }; 'workflow-pre-execute': { @@ -38,14 +78,21 @@ export type Event = { 'workflow-post-execute': { executionId: string; - success: boolean; userId?: string; - workflowId: string; - isManual: boolean; - workflowName: string; - metadata?: Record; + workflow: IWorkflowBase; + runData?: IRun; }; + 'workflow-sharing-updated': { + workflowId: string; + userIdSharer: string; + userIdList: string[]; + }; + + // #endregion + + // #region Node + 'node-pre-execute': { executionId: string; workflow: IWorkflowBase; @@ -58,13 +105,30 @@ export type Event = { nodeName: string; }; + // #endregion + + // #region User + + 'user-submitted-personalization-survey': { + userId: string; + answers: IPersonalizationSurveyAnswersV4; + }; + 'user-deleted': { user: UserLike; + publicApi: boolean; + targetUserOldStatus: 'active' | 'invited'; + migrationStrategy?: 'transfer_data' | 'delete_data'; + targetUserId?: string; + migrationUserId?: string; }; 'user-invited': { user: UserLike; targetUserId: string[]; + publicApi: boolean; + emailSent: boolean; + inviteeRole: string; }; 'user-reinvited': { @@ -79,6 +143,8 @@ export type Event = { 'user-signed-up': { user: UserLike; + userType: AuthProviderType; + wasDisabledLdapUser: boolean; }; 'user-logged-in': { @@ -92,6 +158,47 @@ export type Event = { reason?: string; }; + 'user-changed-role': { + userId: string; + targetUserId: string; + publicApi: boolean; + targetUserNewRole: string; + }; + + 'user-retrieved-user': { + userId: string; + publicApi: boolean; + }; + + 'user-retrieved-all-users': { + userId: string; + publicApi: boolean; + }; + + 'user-retrieved-execution': { + userId: string; + publicApi: boolean; + }; + + 'user-retrieved-all-executions': { + userId: string; + publicApi: boolean; + }; + + 'user-retrieved-workflow': { + userId: string; + publicApi: boolean; + }; + + 'user-retrieved-all-workflows': { + userId: string; + publicApi: boolean; + }; + + // #endregion + + // #region Click + 'user-invite-email-click': { inviter: UserLike; invitee: UserLike; @@ -105,6 +212,31 @@ export type Event = { user: UserLike; }; + 'user-transactional-email-sent': { + userId: string; + messageType: + | 'Reset password' + | 'New user invite' + | 'Resend invite' + | 'Workflow shared' + | 'Credentials shared'; + publicApi: boolean; + }; + + // #endregion + + // #region Public API + + 'public-api-key-created': { + user: UserLike; + publicApi: boolean; + }; + + 'public-api-key-deleted': { + user: UserLike; + publicApi: boolean; + }; + 'public-api-invoked': { userId: string; path: string; @@ -112,6 +244,10 @@ export type Event = { apiVersion: string; }; + // #endregion + + // #region Email + 'email-failed': { user: UserLike; messageType: @@ -120,39 +256,47 @@ export type Event = { | 'Resend invite' | 'Workflow shared' | 'Credentials shared'; + publicApi: boolean; }; + // #endregion + + // #region Credentials + 'credentials-created': { user: UserLike; - credentialName: string; credentialType: string; credentialId: string; + publicApi: boolean; + projectId?: string; + projectType?: string; }; 'credentials-shared': { user: UserLike; - credentialName: string; credentialType: string; credentialId: string; userIdSharer: string; - userIdsShareesRemoved: string[]; + userIdsShareesAdded: string[]; shareesRemoved: number | null; }; 'credentials-updated': { user: UserLike; - credentialName: string; credentialType: string; credentialId: string; }; 'credentials-deleted': { user: UserLike; - credentialName: string; credentialType: string; credentialId: string; }; + // #endregion + + // #region Community package + 'community-package-installed': { user: UserLike; inputString: string; @@ -184,6 +328,10 @@ export type Event = { packageAuthorEmail?: string; }; + // #endregion + + // #region Execution + 'execution-throttled': { executionId: string; }; @@ -192,6 +340,10 @@ export type Event = { executionId: string; }; + // #endregion + + // #region Project + 'team-project-updated': { userId: string; role: GlobalRole; @@ -215,6 +367,10 @@ export type Event = { role: GlobalRole; }; + // #endregion + + // #region Source control + 'source-control-settings-updated': { branchName: string; readOnlyInstance: boolean; @@ -252,12 +408,24 @@ export type Event = { variablesPushed: number; }; + // #endregion + + // #region License + 'license-renewal-attempted': { success: boolean; }; + // #endregion + + // #region Variable + 'variable-created': {}; + // #endregion + + // #region External secrets + 'external-secrets-provider-settings-saved': { userId?: string; vaultType: string; @@ -266,6 +434,10 @@ export type Event = { errorMessage?: string; }; + // #endregion + + // #region LDAP + 'ldap-general-sync-finished': { type: string; succeeded: boolean; @@ -296,17 +468,5 @@ export type Event = { userId: string; }; - /** - * Events listened to by more than one relay - */ - - 'public-api-key-created': { - user: UserLike; // audit and telemetry - publicApi: boolean; // telemetry only - }; - - 'public-api-key-deleted': { - user: UserLike; // audit and telemetry - publicApi: boolean; // telemetry only - }; + // #endregion }; diff --git a/packages/cli/src/events/telemetry-event-relay.ts b/packages/cli/src/events/telemetry-event-relay.ts new file mode 100644 index 0000000000..dc1dae08a0 --- /dev/null +++ b/packages/cli/src/events/telemetry-event-relay.ts @@ -0,0 +1,1010 @@ +import { Service } from 'typedi'; +import { EventService } from '@/events/event.service'; +import type { RelayEventMap } from '@/events/relay-event-map'; +import { Telemetry } from '../telemetry'; +import config from '@/config'; +import os from 'node:os'; +import { License } from '@/license'; +import { GlobalConfig } from '@n8n/config'; +import { N8N_VERSION } from '@/constants'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import type { ExecutionStatus, INodesGraphResult, ITelemetryTrackProperties } from 'n8n-workflow'; +import { get as pslGet } from 'psl'; +import { TelemetryHelpers } from 'n8n-workflow'; +import { NodeTypes } from '@/node-types'; +import { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflow.repository'; +import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; +import type { IExecutionTrackProperties } from '@/Interfaces'; +import { determineFinalExecutionStatus } from '@/execution-lifecycle-hooks/shared/shared-hook-functions'; +import { EventRelay } from './event-relay'; +import { snakeCase } from 'change-case'; + +@Service() +export class TelemetryEventRelay extends EventRelay { + constructor( + readonly eventService: EventService, + private readonly telemetry: Telemetry, + private readonly license: License, + private readonly globalConfig: GlobalConfig, + private readonly workflowRepository: WorkflowRepository, + private readonly nodeTypes: NodeTypes, + private readonly sharedWorkflowRepository: SharedWorkflowRepository, + private readonly projectRelationRepository: ProjectRelationRepository, + ) { + super(eventService); + } + + async init() { + if (!config.getEnv('diagnostics.enabled')) return; + + await this.telemetry.init(); + + this.setupListeners({ + 'team-project-updated': (event) => this.teamProjectUpdated(event), + 'team-project-deleted': (event) => this.teamProjectDeleted(event), + 'team-project-created': (event) => this.teamProjectCreated(event), + 'source-control-settings-updated': (event) => this.sourceControlSettingsUpdated(event), + 'source-control-user-started-pull-ui': (event) => this.sourceControlUserStartedPullUi(event), + 'source-control-user-finished-pull-ui': (event) => + this.sourceControlUserFinishedPullUi(event), + 'source-control-user-pulled-api': (event) => this.sourceControlUserPulledApi(event), + 'source-control-user-started-push-ui': (event) => this.sourceControlUserStartedPushUi(event), + 'source-control-user-finished-push-ui': (event) => + this.sourceControlUserFinishedPushUi(event), + 'license-renewal-attempted': (event) => this.licenseRenewalAttempted(event), + 'variable-created': () => this.variableCreated(), + 'external-secrets-provider-settings-saved': (event) => + this.externalSecretsProviderSettingsSaved(event), + 'public-api-invoked': (event) => this.publicApiInvoked(event), + 'public-api-key-created': (event) => this.publicApiKeyCreated(event), + 'public-api-key-deleted': (event) => this.publicApiKeyDeleted(event), + 'community-package-installed': (event) => this.communityPackageInstalled(event), + 'community-package-updated': (event) => this.communityPackageUpdated(event), + 'community-package-deleted': (event) => this.communityPackageDeleted(event), + 'credentials-created': (event) => this.credentialsCreated(event), + 'credentials-shared': (event) => this.credentialsShared(event), + 'credentials-updated': (event) => this.credentialsUpdated(event), + 'credentials-deleted': (event) => this.credentialsDeleted(event), + 'ldap-general-sync-finished': (event) => this.ldapGeneralSyncFinished(event), + 'ldap-settings-updated': (event) => this.ldapSettingsUpdated(event), + 'ldap-login-sync-failed': (event) => this.ldapLoginSyncFailed(event), + 'login-failed-due-to-ldap-disabled': (event) => this.loginFailedDueToLdapDisabled(event), + 'workflow-created': (event) => this.workflowCreated(event), + 'workflow-deleted': (event) => this.workflowDeleted(event), + 'workflow-sharing-updated': (event) => this.workflowSharingUpdated(event), + 'workflow-saved': async (event) => await this.workflowSaved(event), + 'server-started': async () => await this.serverStarted(), + 'session-started': (event) => this.sessionStarted(event), + 'instance-stopped': () => this.instanceStopped(), + 'instance-owner-setup': async (event) => await this.instanceOwnerSetup(event), + 'first-production-workflow-succeeded': (event) => + this.firstProductionWorkflowSucceeded(event), + 'first-workflow-data-loaded': (event) => this.firstWorkflowDataLoaded(event), + 'workflow-post-execute': async (event) => await this.workflowPostExecute(event), + 'user-changed-role': (event) => this.userChangedRole(event), + 'user-retrieved-user': (event) => this.userRetrievedUser(event), + 'user-retrieved-all-users': (event) => this.userRetrievedAllUsers(event), + 'user-retrieved-execution': (event) => this.userRetrievedExecution(event), + 'user-retrieved-all-executions': (event) => this.userRetrievedAllExecutions(event), + 'user-retrieved-workflow': (event) => this.userRetrievedWorkflow(event), + 'user-retrieved-all-workflows': (event) => this.userRetrievedAllWorkflows(event), + 'user-updated': (event) => this.userUpdated(event), + 'user-deleted': (event) => this.userDeleted(event), + 'user-invited': (event) => this.userInvited(event), + 'user-signed-up': (event) => this.userSignedUp(event), + 'user-submitted-personalization-survey': (event) => + this.userSubmittedPersonalizationSurvey(event), + 'email-failed': (event) => this.emailFailed(event), + 'user-transactional-email-sent': (event) => this.userTransactionalEmailSent(event), + 'user-invite-email-click': (event) => this.userInviteEmailClick(event), + 'user-password-reset-email-click': (event) => this.userPasswordResetEmailClick(event), + 'user-password-reset-request-click': (event) => this.userPasswordResetRequestClick(event), + }); + } + + // #endregion + + // #region Team + + private teamProjectUpdated({ + userId, + role, + members, + projectId, + }: RelayEventMap['team-project-updated']) { + this.telemetry.track('Project settings updated', { + user_id: userId, + role, + // eslint-disable-next-line @typescript-eslint/no-shadow + members: members.map(({ userId: user_id, role }) => ({ user_id, role })), + project_id: projectId, + }); + } + + private teamProjectDeleted({ + userId, + role, + projectId, + removalType, + targetProjectId, + }: RelayEventMap['team-project-deleted']) { + this.telemetry.track('User deleted project', { + user_id: userId, + role, + project_id: projectId, + removal_type: removalType, + target_project_id: targetProjectId, + }); + } + + private teamProjectCreated({ userId, role }: RelayEventMap['team-project-created']) { + this.telemetry.track('User created project', { + user_id: userId, + role, + }); + } + + // #endregion + + // #region Source control + + private sourceControlSettingsUpdated({ + branchName, + readOnlyInstance, + repoType, + connected, + }: RelayEventMap['source-control-settings-updated']) { + this.telemetry.track('User updated source control settings', { + branch_name: branchName, + read_only_instance: readOnlyInstance, + repo_type: repoType, + connected, + }); + } + + private sourceControlUserStartedPullUi({ + workflowUpdates, + workflowConflicts, + credConflicts, + }: RelayEventMap['source-control-user-started-pull-ui']) { + this.telemetry.track('User started pull via UI', { + workflow_updates: workflowUpdates, + workflow_conflicts: workflowConflicts, + cred_conflicts: credConflicts, + }); + } + + private sourceControlUserFinishedPullUi({ + workflowUpdates, + }: RelayEventMap['source-control-user-finished-pull-ui']) { + this.telemetry.track('User finished pull via UI', { + workflow_updates: workflowUpdates, + }); + } + + private sourceControlUserPulledApi({ + workflowUpdates, + forced, + }: RelayEventMap['source-control-user-pulled-api']) { + this.telemetry.track('User pulled via API', { + workflow_updates: workflowUpdates, + forced, + }); + } + + private sourceControlUserStartedPushUi({ + workflowsEligible, + workflowsEligibleWithConflicts, + credsEligible, + credsEligibleWithConflicts, + variablesEligible, + }: RelayEventMap['source-control-user-started-push-ui']) { + this.telemetry.track('User started push via UI', { + workflows_eligible: workflowsEligible, + workflows_eligible_with_conflicts: workflowsEligibleWithConflicts, + creds_eligible: credsEligible, + creds_eligible_with_conflicts: credsEligibleWithConflicts, + variables_eligible: variablesEligible, + }); + } + + private sourceControlUserFinishedPushUi({ + workflowsEligible, + workflowsPushed, + credsPushed, + variablesPushed, + }: RelayEventMap['source-control-user-finished-push-ui']) { + this.telemetry.track('User finished push via UI', { + workflows_eligible: workflowsEligible, + workflows_pushed: workflowsPushed, + creds_pushed: credsPushed, + variables_pushed: variablesPushed, + }); + } + + // #endregion + + // #region License + + private licenseRenewalAttempted({ success }: RelayEventMap['license-renewal-attempted']) { + this.telemetry.track('Instance attempted to refresh license', { + success, + }); + } + + // #endregion + + // #region Variable + + private variableCreated() { + this.telemetry.track('User created variable'); + } + + // #endregion + + // #region External secrets + + private externalSecretsProviderSettingsSaved({ + userId, + vaultType, + isValid, + isNew, + errorMessage, + }: RelayEventMap['external-secrets-provider-settings-saved']) { + this.telemetry.track('User updated external secrets settings', { + user_id: userId, + vault_type: vaultType, + is_valid: isValid, + is_new: isNew, + error_message: errorMessage, + }); + } + + // #endregion + + // #region Public API + + private publicApiInvoked({ + userId, + path, + method, + apiVersion, + }: RelayEventMap['public-api-invoked']) { + this.telemetry.track('User invoked API', { + user_id: userId, + path, + method, + api_version: apiVersion, + }); + } + + private publicApiKeyCreated(event: RelayEventMap['public-api-key-created']) { + const { user, publicApi } = event; + + this.telemetry.track('API key created', { + user_id: user.id, + public_api: publicApi, + }); + } + + private publicApiKeyDeleted(event: RelayEventMap['public-api-key-deleted']) { + const { user, publicApi } = event; + + this.telemetry.track('API key deleted', { + user_id: user.id, + public_api: publicApi, + }); + } + + // #endregion + + // #region Community package + + private communityPackageInstalled({ + user, + inputString, + packageName, + success, + packageVersion, + packageNodeNames, + packageAuthor, + packageAuthorEmail, + failureReason, + }: RelayEventMap['community-package-installed']) { + this.telemetry.track('cnr package install finished', { + user_id: user.id, + input_string: inputString, + package_name: packageName, + success, + package_version: packageVersion, + package_node_names: packageNodeNames, + package_author: packageAuthor, + package_author_email: packageAuthorEmail, + failure_reason: failureReason, + }); + } + + private communityPackageUpdated({ + user, + packageName, + packageVersionCurrent, + packageVersionNew, + packageNodeNames, + packageAuthor, + packageAuthorEmail, + }: RelayEventMap['community-package-updated']) { + this.telemetry.track('cnr package updated', { + user_id: user.id, + package_name: packageName, + package_version_current: packageVersionCurrent, + package_version_new: packageVersionNew, + package_node_names: packageNodeNames, + package_author: packageAuthor, + package_author_email: packageAuthorEmail, + }); + } + + private communityPackageDeleted({ + user, + packageName, + packageVersion, + packageNodeNames, + packageAuthor, + packageAuthorEmail, + }: RelayEventMap['community-package-deleted']) { + this.telemetry.track('cnr package deleted', { + user_id: user.id, + package_name: packageName, + package_version: packageVersion, + package_node_names: packageNodeNames, + package_author: packageAuthor, + package_author_email: packageAuthorEmail, + }); + } + + // #endregion + + // #region Credentials + + private credentialsCreated({ + user, + credentialType, + credentialId, + projectId, + projectType, + }: RelayEventMap['credentials-created']) { + this.telemetry.track('User created credentials', { + user_id: user.id, + credential_type: credentialType, + credential_id: credentialId, + project_id: projectId, + project_type: projectType, + }); + } + + private credentialsShared({ + user, + credentialType, + credentialId, + userIdSharer, + userIdsShareesAdded, + shareesRemoved, + }: RelayEventMap['credentials-shared']) { + this.telemetry.track('User updated cred sharing', { + user_id: user.id, + credential_type: credentialType, + credential_id: credentialId, + user_id_sharer: userIdSharer, + user_ids_sharees_added: userIdsShareesAdded, + sharees_removed: shareesRemoved, + }); + } + + private credentialsUpdated({ + user, + credentialId, + credentialType, + }: RelayEventMap['credentials-updated']) { + this.telemetry.track('User updated credentials', { + user_id: user.id, + credential_type: credentialType, + credential_id: credentialId, + }); + } + + private credentialsDeleted({ + user, + credentialId, + credentialType, + }: RelayEventMap['credentials-deleted']) { + this.telemetry.track('User deleted credentials', { + user_id: user.id, + credential_type: credentialType, + credential_id: credentialId, + }); + } + + // #endregion + + // #region LDAP + + private ldapGeneralSyncFinished({ + type, + succeeded, + usersSynced, + error, + }: RelayEventMap['ldap-general-sync-finished']) { + this.telemetry.track('Ldap general sync finished', { + type, + succeeded, + users_synced: usersSynced, + error, + }); + } + + private ldapSettingsUpdated({ + userId, + loginIdAttribute, + firstNameAttribute, + lastNameAttribute, + emailAttribute, + ldapIdAttribute, + searchPageSize, + searchTimeout, + synchronizationEnabled, + synchronizationInterval, + loginLabel, + loginEnabled, + }: RelayEventMap['ldap-settings-updated']) { + this.telemetry.track('User updated Ldap settings', { + user_id: userId, + loginIdAttribute, + firstNameAttribute, + lastNameAttribute, + emailAttribute, + ldapIdAttribute, + searchPageSize, + searchTimeout, + synchronizationEnabled, + synchronizationInterval, + loginLabel, + loginEnabled, + }); + } + + private ldapLoginSyncFailed({ error }: RelayEventMap['ldap-login-sync-failed']) { + this.telemetry.track('Ldap login sync failed', { error }); + } + + private loginFailedDueToLdapDisabled({ + userId, + }: RelayEventMap['login-failed-due-to-ldap-disabled']) { + this.telemetry.track('User login failed since ldap disabled', { user_ud: userId }); + } + + // #endregion + + // #region Workflow + + private workflowCreated({ + user, + workflow, + publicApi, + projectId, + projectType, + }: RelayEventMap['workflow-created']) { + const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + + this.telemetry.track('User created workflow', { + user_id: user.id, + workflow_id: workflow.id, + node_graph_string: JSON.stringify(nodeGraph), + public_api: publicApi, + project_id: projectId, + project_type: projectType, + }); + } + + private workflowDeleted({ user, workflowId, publicApi }: RelayEventMap['workflow-deleted']) { + this.telemetry.track('User deleted workflow', { + user_id: user.id, + workflow_id: workflowId, + public_api: publicApi, + }); + } + + private workflowSharingUpdated({ + workflowId, + userIdSharer, + userIdList, + }: RelayEventMap['workflow-sharing-updated']) { + this.telemetry.track('User updated workflow sharing', { + workflow_id: workflowId, + user_id_sharer: userIdSharer, + user_id_list: userIdList, + }); + } + + private async workflowSaved({ user, workflow, publicApi }: RelayEventMap['workflow-saved']) { + const isCloudDeployment = config.getEnv('deployment.type') === 'cloud'; + + const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, { + isCloudDeployment, + }); + + let userRole: 'owner' | 'sharee' | 'member' | undefined = undefined; + const role = await this.sharedWorkflowRepository.findSharingRole(user.id, workflow.id); + if (role) { + userRole = role === 'workflow:owner' ? 'owner' : 'sharee'; + } else { + const workflowOwner = await this.sharedWorkflowRepository.getWorkflowOwningProject( + workflow.id, + ); + + if (workflowOwner) { + const projectRole = await this.projectRelationRepository.findProjectRole({ + userId: user.id, + projectId: workflowOwner.id, + }); + + if (projectRole && projectRole !== 'project:personalOwner') { + userRole = 'member'; + } + } + } + + const notesCount = Object.keys(nodeGraph.notes).length; + const overlappingCount = Object.values(nodeGraph.notes).filter( + (note) => note.overlapping, + ).length; + + this.telemetry.track('User saved workflow', { + user_id: user.id, + workflow_id: workflow.id, + node_graph_string: JSON.stringify(nodeGraph), + notes_count_overlapping: overlappingCount, + notes_count_non_overlapping: notesCount - overlappingCount, + version_cli: N8N_VERSION, + num_tags: workflow.tags?.length ?? 0, + public_api: publicApi, + sharing_role: userRole, + }); + } + + // eslint-disable-next-line complexity + private async workflowPostExecute({ + workflow, + runData, + userId, + }: RelayEventMap['workflow-post-execute']) { + if (!workflow.id) { + return; + } + + if (runData?.status === 'waiting') { + // No need to send telemetry or logs when the workflow hasn't finished yet. + return; + } + + const telemetryProperties: IExecutionTrackProperties = { + workflow_id: workflow.id, + is_manual: false, + version_cli: N8N_VERSION, + success: false, + }; + + if (userId) { + telemetryProperties.user_id = userId; + } + + if (runData?.data.resultData.error?.message?.includes('canceled')) { + runData.status = 'canceled'; + } + + telemetryProperties.success = !!runData?.finished; + + // const executionStatus: ExecutionStatus = runData?.status ?? 'unknown'; + const executionStatus: ExecutionStatus = runData + ? determineFinalExecutionStatus(runData) + : 'unknown'; + + if (runData !== undefined) { + telemetryProperties.execution_mode = runData.mode; + telemetryProperties.is_manual = runData.mode === 'manual'; + + let nodeGraphResult: INodesGraphResult | null = null; + + if (!telemetryProperties.success && runData?.data.resultData.error) { + telemetryProperties.error_message = runData?.data.resultData.error.message; + let errorNodeName = + 'node' in runData?.data.resultData.error + ? runData?.data.resultData.error.node?.name + : undefined; + telemetryProperties.error_node_type = + 'node' in runData?.data.resultData.error + ? runData?.data.resultData.error.node?.type + : undefined; + + if (runData.data.resultData.lastNodeExecuted) { + const lastNode = TelemetryHelpers.getNodeTypeForName( + workflow, + runData.data.resultData.lastNodeExecuted, + ); + + if (lastNode !== undefined) { + telemetryProperties.error_node_type = lastNode.type; + errorNodeName = lastNode.name; + } + } + + if (telemetryProperties.is_manual) { + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + telemetryProperties.node_graph = nodeGraphResult.nodeGraph; + telemetryProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); + + if (errorNodeName) { + telemetryProperties.error_node_id = nodeGraphResult.nameIndices[errorNodeName]; + } + } + } + + if (telemetryProperties.is_manual) { + if (!nodeGraphResult) { + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + } + + let userRole: 'owner' | 'sharee' | undefined = undefined; + if (userId) { + const role = await this.sharedWorkflowRepository.findSharingRole(userId, workflow.id); + if (role) { + userRole = role === 'workflow:owner' ? 'owner' : 'sharee'; + } + } + + const manualExecEventProperties: ITelemetryTrackProperties = { + user_id: userId, + workflow_id: workflow.id, + status: executionStatus, + executionStatus: runData?.status ?? 'unknown', + error_message: telemetryProperties.error_message as string, + error_node_type: telemetryProperties.error_node_type, + node_graph_string: telemetryProperties.node_graph_string as string, + error_node_id: telemetryProperties.error_node_id as string, + webhook_domain: null, + sharing_role: userRole, + }; + + if (!manualExecEventProperties.node_graph_string) { + nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); + manualExecEventProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph); + } + + if (runData.data.startData?.destinationNode) { + const telemetryPayload = { + ...manualExecEventProperties, + node_type: TelemetryHelpers.getNodeTypeForName( + workflow, + runData.data.startData?.destinationNode, + )?.type, + node_id: nodeGraphResult.nameIndices[runData.data.startData?.destinationNode], + }; + + this.telemetry.track('Manual node exec finished', telemetryPayload); + } else { + nodeGraphResult.webhookNodeNames.forEach((name: string) => { + const execJson = runData.data.resultData.runData[name]?.[0]?.data?.main?.[0]?.[0] + ?.json as { headers?: { origin?: string } }; + if (execJson?.headers?.origin && execJson.headers.origin !== '') { + manualExecEventProperties.webhook_domain = pslGet( + execJson.headers.origin.replace(/^https?:\/\//, ''), + ); + } + }); + + this.telemetry.track('Manual workflow exec finished', manualExecEventProperties); + } + } + } + + this.telemetry.trackWorkflowExecution(telemetryProperties); + } + + // #endregion + + // #region Lifecycle + + private async serverStarted() { + const cpus = os.cpus(); + const binaryDataConfig = config.getEnv('binaryDataManager'); + + const isS3Selected = config.getEnv('binaryDataManager.mode') === 's3'; + const isS3Available = config.getEnv('binaryDataManager.availableModes').includes('s3'); + const isS3Licensed = this.license.isBinaryDataS3Licensed(); + const authenticationMethod = config.getEnv('userManagement.authenticationMethod'); + + const info = { + version_cli: N8N_VERSION, + db_type: this.globalConfig.database.type, + n8n_version_notifications_enabled: this.globalConfig.versionNotifications.enabled, + n8n_disable_production_main_process: + this.globalConfig.endpoints.disableProductionWebhooksOnMainProcess, + system_info: { + os: { + type: os.type(), + version: os.version(), + }, + memory: os.totalmem() / 1024, + cpus: { + count: cpus.length, + model: cpus[0].model, + speed: cpus[0].speed, + }, + }, + execution_variables: { + executions_mode: config.getEnv('executions.mode'), + executions_timeout: config.getEnv('executions.timeout'), + executions_timeout_max: config.getEnv('executions.maxTimeout'), + executions_data_save_on_error: config.getEnv('executions.saveDataOnError'), + executions_data_save_on_success: config.getEnv('executions.saveDataOnSuccess'), + executions_data_save_on_progress: config.getEnv('executions.saveExecutionProgress'), + executions_data_save_manual_executions: config.getEnv( + 'executions.saveDataManualExecutions', + ), + executions_data_prune: config.getEnv('executions.pruneData'), + executions_data_max_age: config.getEnv('executions.pruneDataMaxAge'), + }, + n8n_deployment_type: config.getEnv('deployment.type'), + n8n_binary_data_mode: binaryDataConfig.mode, + smtp_set_up: this.globalConfig.userManagement.emails.mode === 'smtp', + ldap_allowed: authenticationMethod === 'ldap', + saml_enabled: authenticationMethod === 'saml', + license_plan_name: this.license.getPlanName(), + license_tenant_id: config.getEnv('license.tenantId'), + binary_data_s3: isS3Available && isS3Selected && isS3Licensed, + multi_main_setup_enabled: config.getEnv('multiMainSetup.enabled'), + metrics: { + metrics_enabled: this.globalConfig.endpoints.metrics.enable, + metrics_category_default: this.globalConfig.endpoints.metrics.includeDefaultMetrics, + metrics_category_routes: this.globalConfig.endpoints.metrics.includeApiEndpoints, + metrics_category_cache: this.globalConfig.endpoints.metrics.includeCacheMetrics, + metrics_category_logs: this.globalConfig.endpoints.metrics.includeMessageEventBusMetrics, + }, + }; + + const firstWorkflow = await this.workflowRepository.findOne({ + select: ['createdAt'], + order: { createdAt: 'ASC' }, + where: {}, + }); + + this.telemetry.identify(info); + this.telemetry.track('Instance started', { + ...info, + earliest_workflow_created: firstWorkflow?.createdAt, + }); + } + + private sessionStarted({ pushRef }: RelayEventMap['session-started']) { + this.telemetry.track('Session started', { session_id: pushRef }); + } + + private instanceStopped() { + this.telemetry.track('User instance stopped'); + } + + private async instanceOwnerSetup({ userId }: RelayEventMap['instance-owner-setup']) { + this.telemetry.track('Owner finished instance setup', { user_id: userId }); + } + + private firstProductionWorkflowSucceeded({ + projectId, + workflowId, + userId, + }: RelayEventMap['first-production-workflow-succeeded']) { + this.telemetry.track('Workflow first prod success', { + project_id: projectId, + workflow_id: workflowId, + user_id: userId, + }); + } + + private firstWorkflowDataLoaded({ + userId, + workflowId, + nodeType, + nodeId, + credentialType, + credentialId, + }: RelayEventMap['first-workflow-data-loaded']) { + this.telemetry.track('Workflow first data fetched', { + user_id: userId, + workflow_id: workflowId, + node_type: nodeType, + node_id: nodeId, + credential_type: credentialType, + credential_id: credentialId, + }); + } + + // #endregion + + // #region User + + private userChangedRole({ + userId, + targetUserId, + targetUserNewRole, + publicApi, + }: RelayEventMap['user-changed-role']) { + this.telemetry.track('User changed role', { + user_id: userId, + target_user_id: targetUserId, + target_user_new_role: targetUserNewRole, + public_api: publicApi, + }); + } + + private userRetrievedUser({ userId, publicApi }: RelayEventMap['user-retrieved-user']) { + this.telemetry.track('User retrieved user', { + user_id: userId, + public_api: publicApi, + }); + } + + private userRetrievedAllUsers({ userId, publicApi }: RelayEventMap['user-retrieved-all-users']) { + this.telemetry.track('User retrieved all users', { + user_id: userId, + public_api: publicApi, + }); + } + + private userRetrievedExecution({ userId, publicApi }: RelayEventMap['user-retrieved-execution']) { + this.telemetry.track('User retrieved execution', { + user_id: userId, + public_api: publicApi, + }); + } + + private userRetrievedAllExecutions({ + userId, + publicApi, + }: RelayEventMap['user-retrieved-all-executions']) { + this.telemetry.track('User retrieved all executions', { + user_id: userId, + public_api: publicApi, + }); + } + + private userRetrievedWorkflow({ userId, publicApi }: RelayEventMap['user-retrieved-workflow']) { + this.telemetry.track('User retrieved workflow', { + user_id: userId, + public_api: publicApi, + }); + } + + private userRetrievedAllWorkflows({ + userId, + publicApi, + }: RelayEventMap['user-retrieved-all-workflows']) { + this.telemetry.track('User retrieved all workflows', { + user_id: userId, + public_api: publicApi, + }); + } + + private userUpdated({ user, fieldsChanged }: RelayEventMap['user-updated']) { + this.telemetry.track('User changed personal settings', { + user_id: user.id, + fields_changed: fieldsChanged, + }); + } + + private userDeleted({ + user, + publicApi, + targetUserOldStatus, + migrationStrategy, + targetUserId, + migrationUserId, + }: RelayEventMap['user-deleted']) { + this.telemetry.track('User deleted user', { + user_id: user.id, + public_api: publicApi, + target_user_old_status: targetUserOldStatus, + migration_strategy: migrationStrategy, + target_user_id: targetUserId, + migration_user_id: migrationUserId, + }); + } + + private userInvited({ + user, + targetUserId, + publicApi, + emailSent, + inviteeRole, + }: RelayEventMap['user-invited']) { + this.telemetry.track('User invited new user', { + user_id: user.id, + target_user_id: targetUserId, + public_api: publicApi, + email_sent: emailSent, + invitee_role: inviteeRole, + }); + } + + private userSignedUp({ user, userType, wasDisabledLdapUser }: RelayEventMap['user-signed-up']) { + this.telemetry.track('User signed up', { + user_id: user.id, + user_type: userType, + was_disabled_ldap_user: wasDisabledLdapUser, + }); + } + + private userSubmittedPersonalizationSurvey({ + userId, + answers, + }: RelayEventMap['user-submitted-personalization-survey']) { + const personalizationSurveyData = { user_id: userId } as Record; + + // ESlint is wrong here + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + for (const [camelCaseKey, value] of Object.entries(answers)) { + if (value) { + personalizationSurveyData[snakeCase(camelCaseKey)] = value; + } + } + + this.telemetry.track('User responded to personalization questions', personalizationSurveyData); + } + + // #endregion + + // #region Email + + private emailFailed({ user, messageType, publicApi }: RelayEventMap['email-failed']) { + this.telemetry.track('Instance failed to send transactional email to user', { + user_id: user.id, + message_type: messageType, + public_api: publicApi, + }); + } + + private userTransactionalEmailSent({ + userId, + messageType, + publicApi, + }: RelayEventMap['user-transactional-email-sent']) { + this.telemetry.track('User sent transactional email', { + user_id: userId, + message_type: messageType, + public_api: publicApi, + }); + } + + // #endregion + + // #region Click + + private userInviteEmailClick({ invitee }: RelayEventMap['user-invite-email-click']) { + this.telemetry.track('User clicked invite link from email', { + user_id: invitee.id, + }); + } + + private userPasswordResetEmailClick({ user }: RelayEventMap['user-password-reset-email-click']) { + this.telemetry.track('User clicked password reset link from email', { + user_id: user.id, + }); + } + + private userPasswordResetRequestClick({ + user, + }: RelayEventMap['user-password-reset-request-click']) { + this.telemetry.track('User requested password reset while logged out', { + user_id: user.id, + }); + } + + // #endregion +} diff --git a/packages/cli/test/unit/execution-lifecyle/restoreBinaryDataId.test.ts b/packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts similarity index 95% rename from packages/cli/test/unit/execution-lifecyle/restoreBinaryDataId.test.ts rename to packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts index 3fcdb79c72..58f0825ef7 100644 --- a/packages/cli/test/unit/execution-lifecyle/restoreBinaryDataId.test.ts +++ b/packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts @@ -1,6 +1,6 @@ -import { restoreBinaryDataId } from '@/executionLifecycleHooks/restoreBinaryDataId'; +import { restoreBinaryDataId } from '@/execution-lifecycle-hooks/restore-binary-data-id'; import { BinaryDataService } from 'n8n-core'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import type { IRun } from 'n8n-workflow'; import config from '@/config'; @@ -24,7 +24,7 @@ function toIRun(item?: object) { } function getDataId(run: IRun, kind: 'binary' | 'json') { - // @ts-ignore + // @ts-expect-error The type doesn't have the correct structure return run.data.resultData.runData.myNode[0].data.main[0][0][kind].data.id; } diff --git a/packages/cli/test/unit/execution-lifecyle/saveExecutionProgress.test.ts b/packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts similarity index 91% rename from packages/cli/test/unit/execution-lifecyle/saveExecutionProgress.test.ts rename to packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts index 4235e56ecb..5dc7172659 100644 --- a/packages/cli/test/unit/execution-lifecyle/saveExecutionProgress.test.ts +++ b/packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts @@ -1,8 +1,8 @@ import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { mockInstance } from '../../shared/mocking'; -import { Logger } from '@/Logger'; -import { saveExecutionProgress } from '@/executionLifecycleHooks/saveExecutionProgress'; -import * as fnModule from '@/executionLifecycleHooks/toSaveSettings'; +import { mockInstance } from '@test/mocking'; +import { Logger } from '@/logger'; +import { saveExecutionProgress } from '@/execution-lifecycle-hooks/save-execution-progress'; +import * as fnModule from '@/execution-lifecycle-hooks/to-save-settings'; import { ErrorReporterProxy, type IRunExecutionData, diff --git a/packages/cli/test/unit/execution-lifecyle/toSaveSettings.test.ts b/packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts similarity index 96% rename from packages/cli/test/unit/execution-lifecyle/toSaveSettings.test.ts rename to packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts index 6fc516a0ea..f12c209827 100644 --- a/packages/cli/test/unit/execution-lifecyle/toSaveSettings.test.ts +++ b/packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts @@ -1,5 +1,5 @@ import config from '@/config'; -import { toSaveSettings } from '@/executionLifecycleHooks/toSaveSettings'; +import { toSaveSettings } from '@/execution-lifecycle-hooks/to-save-settings'; afterEach(() => { config.load(config.default); @@ -35,7 +35,7 @@ describe('failed production executions', () => { }); }); -describe('sucessful production executions', () => { +describe('successful production executions', () => { it('should favor workflow settings over defaults', () => { config.set('executions.saveDataOnSuccess', 'none'); diff --git a/packages/cli/src/executionLifecycleHooks/restoreBinaryDataId.ts b/packages/cli/src/execution-lifecycle-hooks/restore-binary-data-id.ts similarity index 98% rename from packages/cli/src/executionLifecycleHooks/restoreBinaryDataId.ts rename to packages/cli/src/execution-lifecycle-hooks/restore-binary-data-id.ts index 84780a785e..d19a6e22a4 100644 --- a/packages/cli/src/executionLifecycleHooks/restoreBinaryDataId.ts +++ b/packages/cli/src/execution-lifecycle-hooks/restore-binary-data-id.ts @@ -3,7 +3,7 @@ import { BinaryDataService } from 'n8n-core'; import type { IRun, WorkflowExecuteMode } from 'n8n-workflow'; import type { BinaryData } from 'n8n-core'; import config from '@/config'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; /** * Whenever the execution ID is not available to the binary data service at the diff --git a/packages/cli/src/executionLifecycleHooks/saveExecutionProgress.ts b/packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts similarity index 96% rename from packages/cli/src/executionLifecycleHooks/saveExecutionProgress.ts rename to packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts index 933b81f788..33b1d9308e 100644 --- a/packages/cli/src/executionLifecycleHooks/saveExecutionProgress.ts +++ b/packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts @@ -3,9 +3,9 @@ import { Container } from 'typedi'; import type { IRunExecutionData, ITaskData, IWorkflowBase } from 'n8n-workflow'; import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { toSaveSettings } from '@/executionLifecycleHooks/toSaveSettings'; +import { toSaveSettings } from '@/execution-lifecycle-hooks/to-save-settings'; export async function saveExecutionProgress( workflowData: IWorkflowBase, diff --git a/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts b/packages/cli/src/execution-lifecycle-hooks/shared/shared-hook-functions.ts similarity index 99% rename from packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts rename to packages/cli/src/execution-lifecycle-hooks/shared/shared-hook-functions.ts index 7a2e401919..2e5436758f 100644 --- a/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts +++ b/packages/cli/src/execution-lifecycle-hooks/shared/shared-hook-functions.ts @@ -5,7 +5,7 @@ import pick from 'lodash/pick'; import { isWorkflowIdValid } from '@/utils'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { ExecutionMetadataService } from '@/services/executionMetadata.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; export function determineFinalExecutionStatus(runData: IRun): ExecutionStatus { const workflowHasCrashed = runData.status === 'crashed'; diff --git a/packages/cli/src/executionLifecycleHooks/toSaveSettings.ts b/packages/cli/src/execution-lifecycle-hooks/to-save-settings.ts similarity index 100% rename from packages/cli/src/executionLifecycleHooks/toSaveSettings.ts rename to packages/cli/src/execution-lifecycle-hooks/to-save-settings.ts diff --git a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts index c1ad38538e..f72c81a3ca 100644 --- a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts +++ b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts @@ -1,6 +1,7 @@ import Container from 'typedi'; import { stringify } from 'flatted'; import { randomInt } from 'n8n-workflow'; +import { InstanceSettings } from 'n8n-core'; import { mockInstance } from '@test/mocking'; import { createWorkflow } from '@test-integration/db/workflows'; @@ -8,11 +9,8 @@ import { createExecution } from '@test-integration/db/executions'; import * as testDb from '@test-integration/testDb'; import { mock } from 'jest-mock-extended'; -import { OrchestrationService } from '@/services/orchestration.service'; -import config from '@/config'; import { ExecutionRecoveryService } from '@/executions/execution-recovery.service'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { InternalHooks } from '@/InternalHooks'; import { Push } from '@/push'; import { ARTIFICIAL_TASK_DATA } from '@/constants'; import { NodeCrashedError } from '@/errors/node-crashed.error'; @@ -21,116 +19,48 @@ import { EventMessageNode } from '@/eventbus/EventMessageClasses/EventMessageNod import { IN_PROGRESS_EXECUTION_DATA, OOM_WORKFLOW } from './constants'; import { setupMessages } from './utils'; -import type { EventService } from '@/eventbus/event.service'; import type { EventMessageTypes as EventMessage } from '@/eventbus/EventMessageClasses'; -import type { Logger } from '@/Logger'; describe('ExecutionRecoveryService', () => { - let push: Push; + const push = mockInstance(Push); + const instanceSettings = new InstanceSettings(); + let executionRecoveryService: ExecutionRecoveryService; - let orchestrationService: OrchestrationService; let executionRepository: ExecutionRepository; beforeAll(async () => { await testDb.init(); - push = mockInstance(Push); executionRepository = Container.get(ExecutionRepository); - orchestrationService = Container.get(OrchestrationService); - mockInstance(InternalHooks); executionRecoveryService = new ExecutionRecoveryService( - mock(), + mock(), + instanceSettings, push, executionRepository, - orchestrationService, - mock(), + mock(), ); }); beforeEach(() => { - config.set('multiMainSetup.instanceType', 'leader'); + instanceSettings.markAsLeader(); }); afterEach(async () => { - config.load(config.default); jest.restoreAllMocks(); await testDb.truncate(['Execution', 'ExecutionData', 'Workflow']); - executionRecoveryService.shutdown(); }); afterAll(async () => { await testDb.terminate(); }); - describe('scheduleQueueRecovery', () => { - describe('queue mode', () => { - it('if leader, should schedule queue recovery', () => { - /** - * Arrange - */ - config.set('executions.mode', 'queue'); - jest.spyOn(orchestrationService, 'isLeader', 'get').mockReturnValue(true); - const scheduleSpy = jest.spyOn(executionRecoveryService, 'scheduleQueueRecovery'); - - /** - * Act - */ - executionRecoveryService.init(); - - /** - * Assert - */ - expect(scheduleSpy).toHaveBeenCalled(); - }); - - it('if follower, should do nothing', () => { - /** - * Arrange - */ - config.set('executions.mode', 'queue'); - jest.spyOn(orchestrationService, 'isLeader', 'get').mockReturnValue(false); - const scheduleSpy = jest.spyOn(executionRecoveryService, 'scheduleQueueRecovery'); - - /** - * Act - */ - executionRecoveryService.init(); - - /** - * Assert - */ - expect(scheduleSpy).not.toHaveBeenCalled(); - }); - }); - - describe('regular mode', () => { - it('should do nothing', () => { - /** - * Arrange - */ - config.set('executions.mode', 'regular'); - const scheduleSpy = jest.spyOn(executionRecoveryService, 'scheduleQueueRecovery'); - - /** - * Act - */ - executionRecoveryService.init(); - - /** - * Assert - */ - expect(scheduleSpy).not.toHaveBeenCalled(); - }); - }); - }); - describe('recoverFromLogs', () => { describe('if follower', () => { test('should do nothing', async () => { /** * Arrange */ - config.set('multiMainSetup.instanceType', 'follower'); + instanceSettings.markAsFollower(); // @ts-expect-error Private method const amendSpy = jest.spyOn(executionRecoveryService, 'amend'); const messages = setupMessages('123', 'Some workflow'); diff --git a/packages/cli/src/executions/__tests__/execution.service.test.ts b/packages/cli/src/executions/__tests__/execution.service.test.ts index 567e0c9758..99370c1e1b 100644 --- a/packages/cli/src/executions/__tests__/execution.service.test.ts +++ b/packages/cli/src/executions/__tests__/execution.service.test.ts @@ -4,16 +4,18 @@ import config from '@/config'; import { ExecutionService } from '@/executions/execution.service'; import { AbortedExecutionRetryError } from '@/errors/aborted-execution-retry.error'; import { MissingExecutionStopError } from '@/errors/missing-execution-stop.error'; -import type { ActiveExecutions } from '@/ActiveExecutions'; +import type { ActiveExecutions } from '@/active-executions'; import type { IExecutionResponse } from '@/Interfaces'; -import type { Job, Queue } from '@/Queue'; -import type { WaitTracker } from '@/WaitTracker'; +import { ScalingService } from '@/scaling/scaling.service'; +import type { WaitTracker } from '@/wait-tracker'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import type { ExecutionRequest } from '@/executions/execution.types'; import type { ConcurrencyControlService } from '@/concurrency/concurrency-control.service'; +import type { Job } from '@/scaling/types'; +import { mockInstance } from '@test/mocking'; describe('ExecutionService', () => { - const queue = mock(); + const scalingService = mockInstance(ScalingService); const activeExecutions = mock(); const executionRepository = mock(); const waitTracker = mock(); @@ -22,7 +24,6 @@ describe('ExecutionService', () => { const executionService = new ExecutionService( mock(), mock(), - queue, activeExecutions, executionRepository, mock(), @@ -31,6 +32,7 @@ describe('ExecutionService', () => { mock(), concurrencyControl, mock(), + mock(), ); beforeEach(() => { @@ -210,7 +212,7 @@ describe('ExecutionService', () => { expect(concurrencyControl.remove).not.toHaveBeenCalled(); expect(waitTracker.stopExecution).not.toHaveBeenCalled(); - expect(queue.stopJob).not.toHaveBeenCalled(); + expect(scalingService.stopJob).not.toHaveBeenCalled(); }); }); @@ -223,7 +225,8 @@ describe('ExecutionService', () => { const execution = mock({ id: '123', status: 'running' }); executionRepository.findSingleExecution.mockResolvedValue(execution); waitTracker.has.mockReturnValue(false); - queue.findRunningJobBy.mockResolvedValue(mock()); + const job = mock({ data: { executionId: '123' } }); + scalingService.findJobsByStatus.mockResolvedValue([job]); executionRepository.stopDuringRun.mockResolvedValue(mock()); /** @@ -236,8 +239,8 @@ describe('ExecutionService', () => { */ expect(waitTracker.stopExecution).not.toHaveBeenCalled(); expect(activeExecutions.stopExecution).toHaveBeenCalled(); - expect(queue.findRunningJobBy).toBeCalledWith({ executionId: execution.id }); - expect(queue.stopJob).toHaveBeenCalled(); + expect(scalingService.findJobsByStatus).toHaveBeenCalled(); + expect(scalingService.stopJob).toHaveBeenCalled(); expect(executionRepository.stopDuringRun).toHaveBeenCalled(); }); @@ -249,7 +252,8 @@ describe('ExecutionService', () => { const execution = mock({ id: '123', status: 'waiting' }); executionRepository.findSingleExecution.mockResolvedValue(execution); waitTracker.has.mockReturnValue(true); - queue.findRunningJobBy.mockResolvedValue(mock()); + const job = mock({ data: { executionId: '123' } }); + scalingService.findJobsByStatus.mockResolvedValue([job]); executionRepository.stopDuringRun.mockResolvedValue(mock()); /** @@ -261,9 +265,8 @@ describe('ExecutionService', () => { * Assert */ expect(waitTracker.stopExecution).toHaveBeenCalledWith(execution.id); - expect(activeExecutions.stopExecution).toHaveBeenCalled(); - expect(queue.findRunningJobBy).toBeCalledWith({ executionId: execution.id }); - expect(queue.stopJob).toHaveBeenCalled(); + expect(scalingService.findJobsByStatus).toHaveBeenCalled(); + expect(scalingService.stopJob).toHaveBeenCalled(); expect(executionRepository.stopDuringRun).toHaveBeenCalled(); }); }); diff --git a/packages/cli/test/unit/controllers/executions.controller.test.ts b/packages/cli/src/executions/__tests__/executions.controller.test.ts similarity index 98% rename from packages/cli/test/unit/controllers/executions.controller.test.ts rename to packages/cli/src/executions/__tests__/executions.controller.test.ts index 2a4c733c5a..acf619a9ea 100644 --- a/packages/cli/test/unit/controllers/executions.controller.test.ts +++ b/packages/cli/src/executions/__tests__/executions.controller.test.ts @@ -3,7 +3,7 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { ExecutionsController } from '@/executions/executions.controller'; import type { ExecutionRequest, ExecutionSummaries } from '@/executions/execution.types'; import type { ExecutionService } from '@/executions/execution.service'; -import type { WorkflowSharingService } from '@/workflows/workflowSharing.service'; +import type { WorkflowSharingService } from '@/workflows/workflow-sharing.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; describe('ExecutionsController', () => { @@ -74,6 +74,8 @@ describe('ExecutionsController', () => { }, ]; + executionService.findRangeWithCount.mockResolvedValue(NO_EXECUTIONS); + describe('if either status or range provided', () => { test.each(QUERIES_WITH_EITHER_STATUS_OR_RANGE)( 'should fetch executions per query', diff --git a/packages/cli/test/unit/middleware/executions/parse-range-query.middleware.test.ts b/packages/cli/src/executions/__tests__/parse-range-query.middleware.test.ts similarity index 100% rename from packages/cli/test/unit/middleware/executions/parse-range-query.middleware.test.ts rename to packages/cli/src/executions/__tests__/parse-range-query.middleware.test.ts diff --git a/packages/cli/src/executions/execution-recovery.service.ts b/packages/cli/src/executions/execution-recovery.service.ts index 4b4d90de54..ffe4e04a45 100644 --- a/packages/cli/src/executions/execution-recovery.service.ts +++ b/packages/cli/src/executions/execution-recovery.service.ts @@ -1,22 +1,18 @@ -import Container, { Service } from 'typedi'; +import { Service } from 'typedi'; import { Push } from '@/push'; -import { jsonStringify, sleep } from 'n8n-workflow'; +import { sleep } from 'n8n-workflow'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { getWorkflowHooksMain } from '@/WorkflowExecuteAdditionalData'; // @TODO: Dependency cycle -import { InternalHooks } from '@/InternalHooks'; // @TODO: Dependency cycle if injected +import { getWorkflowHooksMain } from '@/workflow-execute-additional-data'; // @TODO: Dependency cycle import type { DateTime } from 'luxon'; import type { IRun, ITaskData } from 'n8n-workflow'; +import { InstanceSettings } from 'n8n-core'; import type { EventMessageTypes } from '../eventbus/EventMessageClasses'; import type { IExecutionResponse } from '@/Interfaces'; import { NodeCrashedError } from '@/errors/node-crashed.error'; import { WorkflowCrashedError } from '@/errors/workflow-crashed.error'; import { ARTIFICIAL_TASK_DATA } from '@/constants'; -import { Logger } from '@/Logger'; -import config from '@/config'; -import { OnShutdown } from '@/decorators/OnShutdown'; -import type { QueueRecoverySettings } from './execution.types'; -import { OrchestrationService } from '@/services/orchestration.service'; -import { EventService } from '@/eventbus/event.service'; +import { Logger } from '@/logger'; +import { EventService } from '@/events/event.service'; /** * Service for recovering key properties in executions. @@ -25,41 +21,17 @@ import { EventService } from '@/eventbus/event.service'; export class ExecutionRecoveryService { constructor( private readonly logger: Logger, + private readonly instanceSettings: InstanceSettings, private readonly push: Push, private readonly executionRepository: ExecutionRepository, - private readonly orchestrationService: OrchestrationService, private readonly eventService: EventService, ) {} - /** - * @important Requires `OrchestrationService` to be initialized on queue mode. - */ - init() { - if (config.getEnv('executions.mode') === 'regular') return; - - const { isLeader, isMultiMainSetupEnabled } = this.orchestrationService; - - if (isLeader) this.scheduleQueueRecovery(); - - if (isMultiMainSetupEnabled) { - this.orchestrationService.multiMainSetup - .on('leader-takeover', () => this.scheduleQueueRecovery()) - .on('leader-stepdown', () => this.stopQueueRecovery()); - } - } - - private readonly queueRecoverySettings: QueueRecoverySettings = { - batchSize: config.getEnv('executions.queueRecovery.batchSize'), - waitMs: config.getEnv('executions.queueRecovery.interval') * 60 * 1000, - }; - - private isShuttingDown = false; - /** * Recover key properties of a truncated execution using event logs. */ async recoverFromLogs(executionId: string, messages: EventMessageTypes[]) { - if (this.orchestrationService.isFollower) return; + if (this.instanceSettings.isFollower) return; const amendedExecution = await this.amend(executionId, messages); @@ -81,87 +53,10 @@ export class ExecutionRecoveryService { return amendedExecution; } - /** - * Schedule a cycle to mark dangling executions as crashed in queue mode. - */ - scheduleQueueRecovery(waitMs = this.queueRecoverySettings.waitMs) { - if (!this.shouldScheduleQueueRecovery()) return; - - this.queueRecoverySettings.timeout = setTimeout(async () => { - try { - const nextWaitMs = await this.recoverFromQueue(); - this.scheduleQueueRecovery(nextWaitMs); - } catch (error) { - const msg = this.toErrorMsg(error); - - this.logger.error('[Recovery] Failed to recover dangling executions from queue', { msg }); - this.logger.error('[Recovery] Retrying...'); - - this.scheduleQueueRecovery(); - } - }, waitMs); - - const wait = [this.queueRecoverySettings.waitMs / (60 * 1000), 'min'].join(' '); - - this.logger.debug(`[Recovery] Scheduled queue recovery check for next ${wait}`); - } - - stopQueueRecovery() { - clearTimeout(this.queueRecoverySettings.timeout); - } - - @OnShutdown() - shutdown() { - this.isShuttingDown = true; - this.stopQueueRecovery(); - } - // ---------------------------------- // private // ---------------------------------- - /** - * Mark in-progress executions as `crashed` if stored in DB as `new` or `running` - * but absent from the queue. Return time until next recovery cycle. - */ - private async recoverFromQueue() { - const { waitMs, batchSize } = this.queueRecoverySettings; - - const storedIds = await this.executionRepository.getInProgressExecutionIds(batchSize); - - if (storedIds.length === 0) { - this.logger.debug('[Recovery] Completed queue recovery check, no dangling executions'); - return waitMs; - } - - const { Queue } = await import('@/Queue'); - - const queuedIds = await Container.get(Queue).getInProgressExecutionIds(); - - if (queuedIds.size === 0) { - this.logger.debug('[Recovery] Completed queue recovery check, no dangling executions'); - return waitMs; - } - - const danglingIds = storedIds.filter((id) => !queuedIds.has(id)); - - if (danglingIds.length === 0) { - this.logger.debug('[Recovery] Completed queue recovery check, no dangling executions'); - return waitMs; - } - - await this.executionRepository.markAsCrashed(danglingIds); - - this.logger.info('[Recovery] Completed queue recovery check, recovered dangling executions', { - danglingIds, - }); - - // if this cycle used up the whole batch size, it is possible for there to be - // dangling executions outside this check, so speed up next cycle - - return storedIds.length >= this.queueRecoverySettings.batchSize ? waitMs / 2 : waitMs; - } - /** * Amend `status`, `stoppedAt`, and (if possible) `data` of an execution using event logs. */ @@ -280,22 +175,10 @@ export class ExecutionRecoveryService { private async runHooks(execution: IExecutionResponse) { execution.data ??= { resultData: { runData: {} } }; - await Container.get(InternalHooks).onWorkflowPostExecute(execution.id, execution.workflowData, { - data: execution.data, - finished: false, - mode: execution.mode, - waitTill: execution.waitTill, - startedAt: execution.startedAt, - stoppedAt: execution.stoppedAt, - status: execution.status, - }); - this.eventService.emit('workflow-post-execute', { - workflowId: execution.workflowData.id, - workflowName: execution.workflowData.name, + workflow: execution.workflowData, executionId: execution.id, - success: execution.status === 'success', - isManual: execution.mode === 'manual', + runData: execution, }); const externalHooks = getWorkflowHooksMain( @@ -322,18 +205,4 @@ export class ExecutionRecoveryService { await externalHooks.executeHookFunctions('workflowExecuteAfter', [run]); } - - private toErrorMsg(error: unknown) { - return error instanceof Error - ? error.message - : jsonStringify(error, { replaceCircularRefs: true }); - } - - private shouldScheduleQueueRecovery() { - return ( - config.getEnv('executions.mode') === 'queue' && - config.getEnv('multiMainSetup.instanceType') === 'leader' && - !this.isShuttingDown - ); - } } diff --git a/packages/cli/src/executions/execution.service.ts b/packages/cli/src/executions/execution.service.ts index bb8650e99f..61a1dc0db1 100644 --- a/packages/cli/src/executions/execution.service.ts +++ b/packages/cli/src/executions/execution.service.ts @@ -1,4 +1,4 @@ -import { Service } from 'typedi'; +import { Container, Service } from 'typedi'; import { GlobalConfig } from '@n8n/config'; import { validate as jsonSchemaValidate } from 'jsonschema'; import type { @@ -15,7 +15,7 @@ import { Workflow, WorkflowOperationError, } from 'n8n-workflow'; -import { ActiveExecutions } from '@/ActiveExecutions'; +import { ActiveExecutions } from '@/active-executions'; import type { ExecutionPayload, IExecutionFlattedResponse, @@ -23,23 +23,24 @@ import type { IWorkflowDb, IWorkflowExecutionDataProcess, } from '@/Interfaces'; -import { NodeTypes } from '@/NodeTypes'; -import { Queue } from '@/Queue'; +import { NodeTypes } from '@/node-types'; import type { ExecutionRequest, ExecutionSummaries, StopResult } from './execution.types'; -import { WorkflowRunner } from '@/WorkflowRunner'; +import { WorkflowRunner } from '@/workflow-runner'; import type { IGetExecutionsQueryFilter } from '@db/repositories/execution.repository'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import config from '@/config'; -import { WaitTracker } from '@/WaitTracker'; +import { WaitTracker } from '@/wait-tracker'; import { MissingExecutionStopError } from '@/errors/missing-execution-stop.error'; import { QueuedExecutionRetryError } from '@/errors/queued-execution-retry.error'; import { ConcurrencyControlService } from '@/concurrency/concurrency-control.service'; import { AbortedExecutionRetryError } from '@/errors/aborted-execution-retry.error'; -import { License } from '@/License'; +import { License } from '@/license'; +import type { User } from '@/databases/entities/User'; +import { WorkflowSharingService } from '@/workflows/workflow-sharing.service'; export const schemaGetExecutionsQueryFilter = { $id: '/IGetExecutionsQueryFilter', @@ -83,7 +84,6 @@ export class ExecutionService { constructor( private readonly globalConfig: GlobalConfig, private readonly logger: Logger, - private readonly queue: Queue, private readonly activeExecutions: ActiveExecutions, private readonly executionRepository: ExecutionRepository, private readonly workflowRepository: WorkflowRepository, @@ -92,6 +92,7 @@ export class ExecutionService { private readonly workflowRunner: WorkflowRunner, private readonly concurrencyControl: ConcurrencyControlService, private readonly license: License, + private readonly workflowSharingService: WorkflowSharingService, ) {} async findOne( @@ -468,14 +469,30 @@ export class ExecutionService { this.waitTracker.stopExecution(execution.id); } - const job = await this.queue.findRunningJobBy({ executionId: execution.id }); + const { ScalingService } = await import('@/scaling/scaling.service'); + const scalingService = Container.get(ScalingService); + const jobs = await scalingService.findJobsByStatus(['active', 'waiting']); + + const job = jobs.find(({ data }) => data.executionId === execution.id); if (job) { - await this.queue.stopJob(job); + await scalingService.stopJob(job); } else { this.logger.debug('Job to stop not in queue', { executionId: execution.id }); } return await this.executionRepository.stopDuringRun(execution); } + + async addScopes(user: User, summaries: ExecutionSummaries.ExecutionSummaryWithScopes[]) { + const workflowIds = [...new Set(summaries.map((s) => s.workflowId))]; + + const scopes = Object.fromEntries( + await this.workflowSharingService.getSharedWorkflowScopes(workflowIds, user), + ); + + for (const s of summaries) { + s.scopes = scopes[s.workflowId] ?? []; + } + } } diff --git a/packages/cli/src/executions/execution.types.ts b/packages/cli/src/executions/execution.types.ts index 7e8872bf1b..15c27261fc 100644 --- a/packages/cli/src/executions/execution.types.ts +++ b/packages/cli/src/executions/execution.types.ts @@ -1,6 +1,12 @@ import type { ExecutionEntity } from '@/databases/entities/ExecutionEntity'; import type { AuthenticatedRequest } from '@/requests'; -import type { ExecutionStatus, IDataObject, WorkflowExecuteMode } from 'n8n-workflow'; +import type { Scope } from '@n8n/permissions'; +import type { + ExecutionStatus, + ExecutionSummary, + IDataObject, + WorkflowExecuteMode, +} from 'n8n-workflow'; export declare namespace ExecutionRequest { namespace QueryParams { @@ -83,25 +89,10 @@ export namespace ExecutionSummaries { stoppedAt?: 'DESC'; }; }; + + export type ExecutionSummaryWithScopes = ExecutionSummary & { scopes: Scope[] }; } -export type QueueRecoverySettings = { - /** - * ID of timeout for next scheduled recovery cycle. - */ - timeout?: NodeJS.Timeout; - - /** - * Number of in-progress executions to check per cycle. - */ - batchSize: number; - - /** - * Time (in milliseconds) to wait before the next cycle. - */ - waitMs: number; -}; - export type StopResult = { mode: WorkflowExecuteMode; startedAt: Date; diff --git a/packages/cli/src/executions/executions.controller.ts b/packages/cli/src/executions/executions.controller.ts index 64b6a54274..23e51efda9 100644 --- a/packages/cli/src/executions/executions.controller.ts +++ b/packages/cli/src/executions/executions.controller.ts @@ -1,9 +1,9 @@ -import { ExecutionRequest } from './execution.types'; +import { ExecutionRequest, type ExecutionSummaries } from './execution.types'; import { ExecutionService } from './execution.service'; import { Get, Post, RestController } from '@/decorators'; import { EnterpriseExecutionsService } from './execution.service.ee'; -import { License } from '@/License'; -import { WorkflowSharingService } from '@/workflows/workflowSharing.service'; +import { License } from '@/license'; +import { WorkflowSharingService } from '@/workflows/workflow-sharing.service'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { parseRangeQuery } from './parse-range-query.middleware'; import type { User } from '@/databases/entities/User'; @@ -53,10 +53,20 @@ export class ExecutionsController { const noRange = !query.range.lastId || !query.range.firstId; if (noStatus && noRange) { - return await this.executionService.findLatestCurrentAndCompleted(query); + const executions = await this.executionService.findLatestCurrentAndCompleted(query); + await this.executionService.addScopes( + req.user, + executions.results as ExecutionSummaries.ExecutionSummaryWithScopes[], + ); + return executions; } - return await this.executionService.findRangeWithCount(query); + const executions = await this.executionService.findRangeWithCount(query); + await this.executionService.addScopes( + req.user, + executions.results as ExecutionSummaries.ExecutionSummaryWithScopes[], + ); + return executions; } @Get('/:id') diff --git a/packages/cli/src/executions/parse-range-query.middleware.ts b/packages/cli/src/executions/parse-range-query.middleware.ts index 528c00f676..b02b012c24 100644 --- a/packages/cli/src/executions/parse-range-query.middleware.ts +++ b/packages/cli/src/executions/parse-range-query.middleware.ts @@ -1,4 +1,4 @@ -import * as ResponseHelper from '@/ResponseHelper'; +import * as ResponseHelper from '@/response-helper'; import type { NextFunction, Response } from 'express'; import type { ExecutionRequest } from './execution.types'; import type { JsonObject } from 'n8n-workflow'; diff --git a/packages/cli/src/ExpressionEvaluator.ts b/packages/cli/src/expression-evaluator.ts similarity index 100% rename from packages/cli/src/ExpressionEvaluator.ts rename to packages/cli/src/expression-evaluator.ts diff --git a/packages/cli/src/ExternalHooks.ts b/packages/cli/src/external-hooks.ts similarity index 100% rename from packages/cli/src/ExternalHooks.ts rename to packages/cli/src/external-hooks.ts diff --git a/packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts b/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts similarity index 91% rename from packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts rename to packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts index cf72688d24..6a0d1cc23e 100644 --- a/packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts +++ b/packages/cli/src/external-secrets/__tests__/external-secrets-manager.ee.test.ts @@ -2,17 +2,16 @@ import { Container } from 'typedi'; import { Cipher } from 'n8n-core'; import { SettingsRepository } from '@db/repositories/settings.repository'; import type { ExternalSecretsSettings } from '@/Interfaces'; -import { License } from '@/License'; -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { ExternalSecretsProviders } from '@/ExternalSecrets/ExternalSecretsProviders.ee'; -import { InternalHooks } from '@/InternalHooks'; -import { mockInstance } from '../../shared/mocking'; +import { License } from '@/license'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; +import { ExternalSecretsProviders } from '@/external-secrets/external-secrets-providers.ee'; +import { mockInstance } from '@test/mocking'; import { DummyProvider, ErrorProvider, FailedProvider, MockProviders, -} from '../../shared/ExternalSecrets/utils'; +} from '@test/ExternalSecrets/utils'; import { mock } from 'jest-mock-extended'; describe('External Secrets Manager', () => { @@ -22,7 +21,6 @@ describe('External Secrets Manager', () => { const mockProvidersInstance = new MockProviders(); const license = mockInstance(License); const settingsRepo = mockInstance(SettingsRepository); - mockInstance(InternalHooks); const cipher = Container.get(Cipher); let providersMock: ExternalSecretsProviders; diff --git a/packages/cli/src/ExternalSecrets/constants.ts b/packages/cli/src/external-secrets/constants.ts similarity index 100% rename from packages/cli/src/ExternalSecrets/constants.ts rename to packages/cli/src/external-secrets/constants.ts diff --git a/packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts b/packages/cli/src/external-secrets/external-secrets-helper.ee.ts similarity index 91% rename from packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts rename to packages/cli/src/external-secrets/external-secrets-helper.ee.ts index 40ca21b937..85997d805e 100644 --- a/packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets-helper.ee.ts @@ -1,4 +1,4 @@ -import { License } from '@/License'; +import { License } from '@/license'; import { GlobalConfig } from '@n8n/config'; import Container from 'typedi'; diff --git a/packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts b/packages/cli/src/external-secrets/external-secrets-manager.ee.ts similarity index 97% rename from packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts rename to packages/cli/src/external-secrets/external-secrets-manager.ee.ts index 03436f3d7a..2281f03538 100644 --- a/packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets-manager.ee.ts @@ -8,14 +8,14 @@ import type { import { Cipher } from 'n8n-core'; import Container, { Service } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { jsonParse, type IDataObject, ApplicationError } from 'n8n-workflow'; import { EXTERNAL_SECRETS_INITIAL_BACKOFF, EXTERNAL_SECRETS_MAX_BACKOFF } from './constants'; -import { License } from '@/License'; -import { EventService } from '@/eventbus/event.service'; -import { updateIntervalTime } from './externalSecretsHelper.ee'; -import { ExternalSecretsProviders } from './ExternalSecretsProviders.ee'; +import { License } from '@/license'; +import { EventService } from '@/events/event.service'; +import { updateIntervalTime } from './external-secrets-helper.ee'; +import { ExternalSecretsProviders } from './external-secrets-providers.ee'; import { OrchestrationService } from '@/services/orchestration.service'; @Service() diff --git a/packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts b/packages/cli/src/external-secrets/external-secrets-providers.ee.ts similarity index 86% rename from packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts rename to packages/cli/src/external-secrets/external-secrets-providers.ee.ts index 33228673c3..274a1d69aa 100644 --- a/packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets-providers.ee.ts @@ -4,6 +4,7 @@ import { InfisicalProvider } from './providers/infisical'; import { VaultProvider } from './providers/vault'; import { AwsSecretsManager } from './providers/aws-secrets/aws-secrets-manager'; import { AzureKeyVault } from './providers/azure-key-vault/azure-key-vault'; +import { GcpSecretsManager } from './providers/gcp-secrets-manager/gcp-secrets-manager'; @Service() export class ExternalSecretsProviders { @@ -12,6 +13,7 @@ export class ExternalSecretsProviders { infisical: InfisicalProvider, vault: VaultProvider, azureKeyVault: AzureKeyVault, + gcpSecretsManager: GcpSecretsManager, }; getProvider(name: string): { new (): SecretsProvider } | null { diff --git a/packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts b/packages/cli/src/external-secrets/external-secrets.controller.ee.ts similarity index 97% rename from packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts rename to packages/cli/src/external-secrets/external-secrets.controller.ee.ts index 86a61b75a0..0b27d856dc 100644 --- a/packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets.controller.ee.ts @@ -1,7 +1,7 @@ import { Get, Post, RestController, GlobalScope } from '@/decorators'; import { ExternalSecretsRequest } from '@/requests'; import { Response } from 'express'; -import { ExternalSecretsService } from './ExternalSecrets.service.ee'; +import { ExternalSecretsService } from './external-secrets.service.ee'; import { ExternalSecretsProviderNotFoundError } from '@/errors/external-secrets-provider-not-found.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; diff --git a/packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts b/packages/cli/src/external-secrets/external-secrets.service.ee.ts similarity index 98% rename from packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts rename to packages/cli/src/external-secrets/external-secrets.service.ee.ts index 0c378540ac..ea9f4ed170 100644 --- a/packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts +++ b/packages/cli/src/external-secrets/external-secrets.service.ee.ts @@ -4,7 +4,7 @@ import type { ExternalSecretsRequest } from '@/requests'; import type { IDataObject } from 'n8n-workflow'; import { deepCopy } from 'n8n-workflow'; import Container, { Service } from 'typedi'; -import { ExternalSecretsManager } from './ExternalSecretsManager.ee'; +import { ExternalSecretsManager } from './external-secrets-manager.ee'; import { ExternalSecretsProviderNotFoundError } from '@/errors/external-secrets-provider-not-found.error'; @Service() diff --git a/packages/cli/src/ExternalSecrets/providers/__tests__/azure-key-vault.test.ts b/packages/cli/src/external-secrets/providers/__tests__/azure-key-vault.test.ts similarity index 93% rename from packages/cli/src/ExternalSecrets/providers/__tests__/azure-key-vault.test.ts rename to packages/cli/src/external-secrets/providers/__tests__/azure-key-vault.test.ts index 329bfca2ac..8bacd6bbb9 100644 --- a/packages/cli/src/ExternalSecrets/providers/__tests__/azure-key-vault.test.ts +++ b/packages/cli/src/external-secrets/providers/__tests__/azure-key-vault.test.ts @@ -37,7 +37,7 @@ describe('AzureKeyVault', () => { yield { name: 'secret1' }; yield { name: 'secret2' }; yield { name: 'secret3' }; // no value - yield { name: '#@&' }; // invalid name + yield { name: '#@&' }; // unsupported name }, })); @@ -65,6 +65,6 @@ describe('AzureKeyVault', () => { expect(azureKeyVault.getSecret('secret1')).toBe('value1'); expect(azureKeyVault.getSecret('secret2')).toBe('value2'); expect(azureKeyVault.getSecret('secret3')).toBeUndefined(); // no value - expect(azureKeyVault.getSecret('#@&')).toBeUndefined(); // invalid name + expect(azureKeyVault.getSecret('#@&')).toBeUndefined(); // unsupported name }); }); diff --git a/packages/cli/src/external-secrets/providers/__tests__/gcp-secrets-manager.test.ts b/packages/cli/src/external-secrets/providers/__tests__/gcp-secrets-manager.test.ts new file mode 100644 index 0000000000..40673041a7 --- /dev/null +++ b/packages/cli/src/external-secrets/providers/__tests__/gcp-secrets-manager.test.ts @@ -0,0 +1,87 @@ +import { mock } from 'jest-mock-extended'; +import { GcpSecretsManager } from '../gcp-secrets-manager/gcp-secrets-manager'; +import type { GcpSecretsManagerContext } from '../gcp-secrets-manager/types'; +import { SecretManagerServiceClient } from '@google-cloud/secret-manager'; +import type { google } from '@google-cloud/secret-manager/build/protos/protos'; + +jest.mock('@google-cloud/secret-manager'); + +type GcpSecretVersionResponse = google.cloud.secretmanager.v1.IAccessSecretVersionResponse; + +describe('GCP Secrets Manager', () => { + const gcpSecretsManager = new GcpSecretsManager(); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should update cached secrets', async () => { + /** + * Arrange + */ + const PROJECT_ID = 'my-project-id'; + + const SECRETS: Record = { + secret1: 'value1', + secret2: 'value2', + secret3: '', // no value + '#@&': 'value', // unsupported name + }; + + await gcpSecretsManager.init( + mock({ + settings: { serviceAccountKey: `{ "project_id": "${PROJECT_ID}" }` }, + }), + ); + + const listSpy = jest + .spyOn(SecretManagerServiceClient.prototype, 'listSecrets') + // @ts-expect-error Partial mock + .mockResolvedValue([ + [ + { name: `projects/${PROJECT_ID}/secrets/secret1` }, + { name: `projects/${PROJECT_ID}/secrets/secret2` }, + { name: `projects/${PROJECT_ID}/secrets/secret3` }, + { name: `projects/${PROJECT_ID}/secrets/#@&` }, + ], + ]); + + const getSpy = jest + .spyOn(SecretManagerServiceClient.prototype, 'accessSecretVersion') + .mockImplementation(async ({ name }: { name: string }) => { + const secretName = name.split('/')[3]; + return [ + { payload: { data: Buffer.from(SECRETS[secretName]) } }, + ] as GcpSecretVersionResponse[]; + }); + + /** + * Act + */ + await gcpSecretsManager.connect(); + await gcpSecretsManager.update(); + + /** + * Assert + */ + expect(listSpy).toHaveBeenCalled(); + + expect(getSpy).toHaveBeenCalledWith({ + name: `projects/${PROJECT_ID}/secrets/secret1/versions/latest`, + }); + expect(getSpy).toHaveBeenCalledWith({ + name: `projects/${PROJECT_ID}/secrets/secret2/versions/latest`, + }); + expect(getSpy).toHaveBeenCalledWith({ + name: `projects/${PROJECT_ID}/secrets/secret3/versions/latest`, + }); + expect(getSpy).not.toHaveBeenCalledWith({ + name: `projects/${PROJECT_ID}/secrets/#@&/versions/latest`, + }); + + expect(gcpSecretsManager.getSecret('secret1')).toBe('value1'); + expect(gcpSecretsManager.getSecret('secret2')).toBe('value2'); + expect(gcpSecretsManager.getSecret('secret3')).toBeUndefined(); // no value + expect(gcpSecretsManager.getSecret('#@&')).toBeUndefined(); // unsupported name + }); +}); diff --git a/packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-client.ts b/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-client.ts similarity index 100% rename from packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-client.ts rename to packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-client.ts diff --git a/packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-manager.ts b/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts similarity index 99% rename from packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-manager.ts rename to packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts index b7b5f52d7b..a4de7d164c 100644 --- a/packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-manager.ts +++ b/packages/cli/src/external-secrets/providers/aws-secrets/aws-secrets-manager.ts @@ -1,6 +1,6 @@ import { AwsSecretsClient } from './aws-secrets-client'; import { UnknownAuthTypeError } from '@/errors/unknown-auth-type.error'; -import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/ExternalSecrets/constants'; +import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; import type { SecretsProvider, SecretsProviderState } from '@/Interfaces'; import type { INodeProperties } from 'n8n-workflow'; import type { AwsSecretsManagerContext } from './types'; diff --git a/packages/cli/src/ExternalSecrets/providers/aws-secrets/types.ts b/packages/cli/src/external-secrets/providers/aws-secrets/types.ts similarity index 100% rename from packages/cli/src/ExternalSecrets/providers/aws-secrets/types.ts rename to packages/cli/src/external-secrets/providers/aws-secrets/types.ts diff --git a/packages/cli/src/ExternalSecrets/providers/azure-key-vault/azure-key-vault.ts b/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts similarity index 96% rename from packages/cli/src/ExternalSecrets/providers/azure-key-vault/azure-key-vault.ts rename to packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts index c51e54e799..ad272abe54 100644 --- a/packages/cli/src/ExternalSecrets/providers/azure-key-vault/azure-key-vault.ts +++ b/packages/cli/src/external-secrets/providers/azure-key-vault/azure-key-vault.ts @@ -3,7 +3,7 @@ import { SecretClient } from '@azure/keyvault-secrets'; import type { SecretsProvider, SecretsProviderState } from '@/Interfaces'; import type { INodeProperties } from 'n8n-workflow'; import type { AzureKeyVaultContext } from './types'; -import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/ExternalSecrets/constants'; +import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; export class AzureKeyVault implements SecretsProvider { name = 'azureKeyVault'; @@ -74,7 +74,7 @@ export class AzureKeyVault implements SecretsProvider { const credential = new ClientSecretCredential(tenantId, clientId, clientSecret); this.client = new SecretClient(`https://${vaultName}.vault.azure.net/`, credential); this.state = 'connected'; - } catch (error) { + } catch { this.state = 'error'; } } @@ -86,7 +86,7 @@ export class AzureKeyVault implements SecretsProvider { await this.client.listPropertiesOfSecrets().next(); return [true]; } catch (error: unknown) { - return [false, error instanceof Error ? error.message : 'unknown error']; + return [false, error instanceof Error ? error.message : 'Unknown error']; } } diff --git a/packages/cli/src/ExternalSecrets/providers/azure-key-vault/types.ts b/packages/cli/src/external-secrets/providers/azure-key-vault/types.ts similarity index 100% rename from packages/cli/src/ExternalSecrets/providers/azure-key-vault/types.ts rename to packages/cli/src/external-secrets/providers/azure-key-vault/types.ts diff --git a/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts new file mode 100644 index 0000000000..ac00245c58 --- /dev/null +++ b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/gcp-secrets-manager.ts @@ -0,0 +1,136 @@ +import { SecretManagerServiceClient as GcpClient } from '@google-cloud/secret-manager'; +import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '@/external-secrets/constants'; +import type { SecretsProvider, SecretsProviderState } from '@/Interfaces'; +import { jsonParse, type INodeProperties } from 'n8n-workflow'; +import type { + GcpSecretsManagerContext, + GcpSecretAccountKey, + RawGcpSecretAccountKey, +} from './types'; + +export class GcpSecretsManager implements SecretsProvider { + name = 'gcpSecretsManager'; + + displayName = 'GCP Secrets Manager'; + + state: SecretsProviderState = 'initializing'; + + properties: INodeProperties[] = [ + DOCS_HELP_NOTICE, + { + displayName: 'Service Account Key', + name: 'serviceAccountKey', + type: 'string', + default: '', + required: true, + typeOptions: { password: true }, + placeholder: 'e.g. { "type": "service_account", "project_id": "gcp-secrets-store", ... }', + hint: 'Content of JSON file downloaded from Google Cloud Console.', + noDataExpression: true, + }, + ]; + + private cachedSecrets: Record = {}; + + private client: GcpClient; + + private settings: GcpSecretAccountKey; + + async init(context: GcpSecretsManagerContext) { + this.settings = this.parseSecretAccountKey(context.settings.serviceAccountKey); + } + + async connect() { + const { projectId, privateKey, clientEmail } = this.settings; + + try { + this.client = new GcpClient({ + credentials: { client_email: clientEmail, private_key: privateKey }, + projectId, + }); + this.state = 'connected'; + } catch { + this.state = 'error'; + } + } + + async test(): Promise<[boolean] | [boolean, string]> { + if (!this.client) return [false, 'Failed to connect to GCP Secrets Manager']; + + try { + await this.client.initialize(); + return [true]; + } catch (error: unknown) { + return [false, error instanceof Error ? error.message : 'Unknown error']; + } + } + + async disconnect() { + // unused + } + + async update() { + const { projectId } = this.settings; + + const [rawSecretNames] = await this.client.listSecrets({ + parent: `projects/${projectId}`, + }); + + const secretNames = rawSecretNames.reduce((acc, cur) => { + if (!cur.name || !EXTERNAL_SECRETS_NAME_REGEX.test(cur.name)) return acc; + + const secretName = cur.name.split('/').pop(); + + if (secretName) acc.push(secretName); + + return acc; + }, []); + + const promises = secretNames.map(async (name) => { + const versions = await this.client.accessSecretVersion({ + name: `projects/${projectId}/secrets/${name}/versions/latest`, + }); + + if (!Array.isArray(versions) || !versions.length) return null; + + const [latestVersion] = versions; + + if (!latestVersion.payload?.data) return null; + + const value = latestVersion.payload.data.toString(); + + if (!value) return null; + + return { name, value }; + }); + + const results = await Promise.all(promises); + + this.cachedSecrets = results.reduce>((acc, cur) => { + if (cur) acc[cur.name] = cur.value; + return acc; + }, {}); + } + + getSecret(name: string) { + return this.cachedSecrets[name]; + } + + hasSecret(name: string) { + return name in this.cachedSecrets; + } + + getSecretNames() { + return Object.keys(this.cachedSecrets); + } + + private parseSecretAccountKey(privateKey: string): GcpSecretAccountKey { + const parsed = jsonParse(privateKey, { fallbackValue: {} }); + + return { + projectId: parsed?.project_id ?? '', + clientEmail: parsed?.client_email ?? '', + privateKey: parsed?.private_key ?? '', + }; + } +} diff --git a/packages/cli/src/external-secrets/providers/gcp-secrets-manager/types.ts b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/types.ts new file mode 100644 index 0000000000..c9611df9c4 --- /dev/null +++ b/packages/cli/src/external-secrets/providers/gcp-secrets-manager/types.ts @@ -0,0 +1,19 @@ +import type { SecretsProviderSettings } from '@/Interfaces'; + +type JsonString = string; + +export type GcpSecretsManagerContext = SecretsProviderSettings<{ + serviceAccountKey: JsonString; +}>; + +export type RawGcpSecretAccountKey = { + project_id?: string; + private_key?: string; + client_email?: string; +}; + +export type GcpSecretAccountKey = { + projectId: string; + clientEmail: string; + privateKey: string; +}; diff --git a/packages/cli/src/ExternalSecrets/providers/infisical.ts b/packages/cli/src/external-secrets/providers/infisical.ts similarity index 84% rename from packages/cli/src/ExternalSecrets/providers/infisical.ts rename to packages/cli/src/external-secrets/providers/infisical.ts index d9a2f93cda..d352d59179 100644 --- a/packages/cli/src/ExternalSecrets/providers/infisical.ts +++ b/packages/cli/src/external-secrets/providers/infisical.ts @@ -3,7 +3,7 @@ import InfisicalClient from 'infisical-node'; import { populateClientWorkspaceConfigsHelper } from 'infisical-node/lib/helpers/key'; import { getServiceTokenData } from 'infisical-node/lib/api/serviceTokenData'; import { ApplicationError, type IDataObject, type INodeProperties } from 'n8n-workflow'; -import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '../constants'; +import { EXTERNAL_SECRETS_NAME_REGEX } from '../constants'; export interface InfisicalSettings { token: string; @@ -24,7 +24,14 @@ interface InfisicalServiceToken { export class InfisicalProvider implements SecretsProvider { properties: INodeProperties[] = [ - DOCS_HELP_NOTICE, + { + displayName: + '

Important information about our infisical integration


From the 30th July, 2024, we will no longer be supporting new connections to inifiscal secrets vault using service tokens. Existing service tokens will remain usable until July, 2025. After that period, we will be removing support for Infisical from our external secrets integrations. You can find out more information about this change on our docs', + name: 'notice', + type: 'notice', + default: '', + noDataExpression: true, + }, { displayName: 'Service Token', name: 'token', diff --git a/packages/cli/src/ExternalSecrets/providers/vault.ts b/packages/cli/src/external-secrets/providers/vault.ts similarity index 99% rename from packages/cli/src/ExternalSecrets/providers/vault.ts rename to packages/cli/src/external-secrets/providers/vault.ts index 6735d26d6d..e69a909481 100644 --- a/packages/cli/src/ExternalSecrets/providers/vault.ts +++ b/packages/cli/src/external-secrets/providers/vault.ts @@ -3,9 +3,9 @@ import { SecretsProvider } from '@/Interfaces'; import type { IDataObject, INodeProperties } from 'n8n-workflow'; import type { AxiosInstance, AxiosResponse } from 'axios'; import axios from 'axios'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { DOCS_HELP_NOTICE, EXTERNAL_SECRETS_NAME_REGEX } from '../constants'; -import { preferGet } from '../externalSecretsHelper.ee'; +import { preferGet } from '../external-secrets-helper.ee'; import { Container } from 'typedi'; type VaultAuthMethod = 'token' | 'usernameAndPassword' | 'appRole'; diff --git a/packages/cli/src/GenericHelpers.ts b/packages/cli/src/generic-helpers.ts similarity index 75% rename from packages/cli/src/GenericHelpers.ts rename to packages/cli/src/generic-helpers.ts index 13762e5dfd..39bdc4c9c1 100644 --- a/packages/cli/src/GenericHelpers.ts +++ b/packages/cli/src/generic-helpers.ts @@ -3,8 +3,13 @@ import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; import type { TagEntity } from '@db/entities/TagEntity'; import type { User } from '@db/entities/User'; -import type { UserRoleChangePayload, UserUpdatePayload } from '@/requests'; +import type { + UserRoleChangePayload, + UserSettingsUpdatePayload, + UserUpdatePayload, +} from '@/requests'; import { BadRequestError } from './errors/response-errors/bad-request.error'; +import type { PersonalizationSurveyAnswersV4 } from './controllers/survey-answers.dto'; export async function validateEntity( entity: @@ -13,7 +18,9 @@ export async function validateEntity( | TagEntity | User | UserUpdatePayload - | UserRoleChangePayload, + | UserRoleChangePayload + | UserSettingsUpdatePayload + | PersonalizationSurveyAnswersV4, ): Promise { const errors = await validate(entity); diff --git a/packages/cli/src/internal-hooks.ts b/packages/cli/src/internal-hooks.ts new file mode 100644 index 0000000000..b3d45b67c0 --- /dev/null +++ b/packages/cli/src/internal-hooks.ts @@ -0,0 +1,24 @@ +import { Service } from 'typedi'; +import { Telemetry } from '@/telemetry'; +import { MessageEventBus } from './eventbus/MessageEventBus/MessageEventBus'; + +/** + * @deprecated Do not add to this class. It will be removed once we remove + * further dep cycles. To add log streaming or telemetry events, use + * `EventService` to emit the event and then use the `LogStreamingEventRelay` or + * `TelemetryEventRelay` to forward them to the event bus or telemetry. + */ +@Service() +export class InternalHooks { + constructor( + private readonly telemetry: Telemetry, + // Can't use @ts-expect-error because only dev time tsconfig considers this as an error, but not build time + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - needed until we decouple telemetry + private readonly _eventBus: MessageEventBus, // needed until we decouple telemetry + ) {} + + async init() { + await this.telemetry.init(); + } +} diff --git a/packages/cli/test/unit/Ldap/helpers.test.ts b/packages/cli/src/ldap/__tests__/helpers.test.ts similarity index 93% rename from packages/cli/test/unit/Ldap/helpers.test.ts rename to packages/cli/src/ldap/__tests__/helpers.test.ts index debb96da38..876a330520 100644 --- a/packages/cli/test/unit/Ldap/helpers.test.ts +++ b/packages/cli/src/ldap/__tests__/helpers.test.ts @@ -1,6 +1,6 @@ import { UserRepository } from '@/databases/repositories/user.repository'; -import { mockInstance } from '../../shared/mocking'; -import * as helpers from '@/Ldap/helpers.ee'; +import { mockInstance } from '@test/mocking'; +import * as helpers from '@/ldap/helpers.ee'; import { AuthIdentity } from '@/databases/entities/AuthIdentity'; import { User } from '@/databases/entities/User'; import { generateNanoId } from '@/databases/utils/generators'; diff --git a/packages/cli/src/Ldap/constants.ts b/packages/cli/src/ldap/constants.ts similarity index 100% rename from packages/cli/src/Ldap/constants.ts rename to packages/cli/src/ldap/constants.ts diff --git a/packages/cli/src/Ldap/helpers.ee.ts b/packages/cli/src/ldap/helpers.ee.ts similarity index 99% rename from packages/cli/src/Ldap/helpers.ee.ts rename to packages/cli/src/ldap/helpers.ee.ts index bf91bb83db..d705fb66fa 100644 --- a/packages/cli/src/Ldap/helpers.ee.ts +++ b/packages/cli/src/ldap/helpers.ee.ts @@ -17,7 +17,7 @@ import { LDAP_LOGIN_LABEL, } from './constants'; import type { ConnectionSecurity, LdapConfig } from './types'; -import { License } from '@/License'; +import { License } from '@/license'; import { UserRepository } from '@db/repositories/user.repository'; import { AuthProviderSyncHistoryRepository } from '@db/repositories/authProviderSyncHistory.repository'; import { AuthIdentityRepository } from '@db/repositories/authIdentity.repository'; diff --git a/packages/cli/src/Ldap/ldap.controller.ee.ts b/packages/cli/src/ldap/ldap.controller.ee.ts similarity index 97% rename from packages/cli/src/Ldap/ldap.controller.ee.ts rename to packages/cli/src/ldap/ldap.controller.ee.ts index 9bdbea39b2..7a56d8049d 100644 --- a/packages/cli/src/Ldap/ldap.controller.ee.ts +++ b/packages/cli/src/ldap/ldap.controller.ee.ts @@ -6,7 +6,7 @@ import { NON_SENSIBLE_LDAP_CONFIG_PROPERTIES } from './constants'; import { getLdapSynchronizations } from './helpers.ee'; import { LdapConfiguration } from './types'; import { LdapService } from './ldap.service.ee'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @RestController('/ldap') export class LdapController { diff --git a/packages/cli/src/Ldap/ldap.service.ee.ts b/packages/cli/src/ldap/ldap.service.ee.ts similarity index 99% rename from packages/cli/src/Ldap/ldap.service.ee.ts rename to packages/cli/src/ldap/ldap.service.ee.ts index 85c8c6c636..c24cade682 100644 --- a/packages/cli/src/Ldap/ldap.service.ee.ts +++ b/packages/cli/src/ldap/ldap.service.ee.ts @@ -11,7 +11,7 @@ import config from '@/config'; import type { User } from '@db/entities/User'; import type { RunningMode, SyncStatus } from '@db/entities/AuthProviderSyncHistory'; import { SettingsRepository } from '@db/repositories/settings.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; @@ -20,7 +20,7 @@ import { isEmailCurrentAuthenticationMethod, isLdapCurrentAuthenticationMethod, setCurrentAuthenticationMethod, -} from '@/sso/ssoHelpers'; +} from '@/sso/sso-helpers'; import type { LdapConfig } from './types'; import { @@ -44,7 +44,7 @@ import { LDAP_LOGIN_ENABLED, LDAP_LOGIN_LABEL, } from './constants'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @Service() export class LdapService { diff --git a/packages/cli/src/Ldap/types.ts b/packages/cli/src/ldap/types.ts similarity index 100% rename from packages/cli/src/Ldap/types.ts rename to packages/cli/src/ldap/types.ts diff --git a/packages/cli/src/License.ts b/packages/cli/src/license.ts similarity index 96% rename from packages/cli/src/License.ts rename to packages/cli/src/license.ts index 25abf0ecd6..57f576c5a3 100644 --- a/packages/cli/src/License.ts +++ b/packages/cli/src/license.ts @@ -2,7 +2,7 @@ import type { TEntitlement, TFeatures, TLicenseBlock } from '@n8n_io/license-sdk import { LicenseManager } from '@n8n_io/license-sdk'; import { InstanceSettings, ObjectStoreService } from 'n8n-core'; import Container, { Service } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import config from '@/config'; import { LICENSE_FEATURES, @@ -16,7 +16,7 @@ import type { BooleanLicenseFeature, N8nInstanceType, NumericLicenseFeature } fr import type { RedisServicePubSubPublisher } from './services/redis/RedisServicePubSubPublisher'; import { RedisService } from './services/redis.service'; import { OrchestrationService } from '@/services/orchestration.service'; -import { OnShutdown } from '@/decorators/OnShutdown'; +import { OnShutdown } from '@/decorators/on-shutdown'; import { LicenseMetricsService } from '@/metrics/license-metrics.service'; type FeatureReturnType = Partial< @@ -55,7 +55,7 @@ export class License { * This ensures the mains do not cause a 429 (too many requests) on license init. */ if (config.getEnv('multiMainSetup.enabled')) { - return autoRenewEnabled && config.getEnv('multiMainSetup.instanceType') === 'leader'; + return autoRenewEnabled && this.instanceSettings.isLeader; } return autoRenewEnabled; @@ -249,6 +249,10 @@ export class License { return this.isFeatureEnabled(LICENSE_FEATURES.SAML); } + isAiAssistantEnabled() { + return this.isFeatureEnabled(LICENSE_FEATURES.AI_ASSISTANT); + } + isAdvancedExecutionFiltersEnabled() { return this.isFeatureEnabled(LICENSE_FEATURES.ADVANCED_EXECUTION_FILTERS); } @@ -305,6 +309,10 @@ export class License { return this.isFeatureEnabled(LICENSE_FEATURES.PROJECT_ROLE_VIEWER); } + isCustomNpmRegistryEnabled() { + return this.isFeatureEnabled(LICENSE_FEATURES.COMMUNITY_NODES_CUSTOM_REGISTRY); + } + getCurrentEntitlements() { return this.manager?.getCurrentEntitlements() ?? []; } diff --git a/packages/cli/test/unit/license/license.service.test.ts b/packages/cli/src/license/__tests__/license.service.test.ts similarity index 95% rename from packages/cli/test/unit/license/license.service.test.ts rename to packages/cli/src/license/__tests__/license.service.test.ts index e28895025f..9db929ec71 100644 --- a/packages/cli/test/unit/license/license.service.test.ts +++ b/packages/cli/src/license/__tests__/license.service.test.ts @@ -1,6 +1,6 @@ import { LicenseErrors, LicenseService } from '@/license/license.service'; -import type { License } from '@/License'; -import type { EventService } from '@/eventbus/event.service'; +import type { License } from '@/license'; +import type { EventService } from '@/events/event.service'; import type { WorkflowRepository } from '@db/repositories/workflow.repository'; import type { TEntitlement } from '@n8n_io/license-sdk'; import { mock } from 'jest-mock-extended'; diff --git a/packages/cli/src/license/license.controller.ts b/packages/cli/src/license/license.controller.ts index 1c0ca8c0d6..945f3650ea 100644 --- a/packages/cli/src/license/license.controller.ts +++ b/packages/cli/src/license/license.controller.ts @@ -1,6 +1,8 @@ import { Get, Post, RestController, GlobalScope } from '@/decorators'; import { AuthenticatedRequest, LicenseRequest } from '@/requests'; import { LicenseService } from './license.service'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import type { AxiosError } from 'axios'; @RestController('/license') export class LicenseController { @@ -14,7 +16,18 @@ export class LicenseController { @Post('/enterprise/request_trial') @GlobalScope('license:manage') async requestEnterpriseTrial(req: AuthenticatedRequest) { - await this.licenseService.requestEnterpriseTrial(req.user); + try { + await this.licenseService.requestEnterpriseTrial(req.user); + } catch (error: unknown) { + if (error instanceof Error) { + const errorMsg = + (error as AxiosError<{ message: string }>).response?.data?.message ?? error.message; + + throw new BadRequestError(errorMsg); + } else { + throw new BadRequestError('Failed to request trial'); + } + } } @Post('/activate') diff --git a/packages/cli/src/license/license.service.ts b/packages/cli/src/license/license.service.ts index 01d2a73c48..6125ed813b 100644 --- a/packages/cli/src/license/license.service.ts +++ b/packages/cli/src/license/license.service.ts @@ -1,9 +1,9 @@ import { Service } from 'typedi'; import axios from 'axios'; -import { Logger } from '@/Logger'; -import { License } from '@/License'; -import { EventService } from '@/eventbus/event.service'; +import { Logger } from '@/logger'; +import { License } from '@/license'; +import { EventService } from '@/events/event.service'; import type { User } from '@db/entities/User'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; diff --git a/packages/cli/src/LoadNodesAndCredentials.ts b/packages/cli/src/load-nodes-and-credentials.ts similarity index 99% rename from packages/cli/src/LoadNodesAndCredentials.ts rename to packages/cli/src/load-nodes-and-credentials.ts index a1f7b0578d..5b2cc284e9 100644 --- a/packages/cli/src/LoadNodesAndCredentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -26,7 +26,7 @@ import { CLI_DIR, inE2ETests, } from '@/constants'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { GlobalConfig } from '@n8n/config'; interface LoadedNodesAndCredentials { diff --git a/packages/cli/src/Logger.ts b/packages/cli/src/logger.ts similarity index 100% rename from packages/cli/src/Logger.ts rename to packages/cli/src/logger.ts diff --git a/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts index 8b89878203..219170ac08 100644 --- a/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts +++ b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts @@ -5,6 +5,8 @@ import { mock } from 'jest-mock-extended'; import { PrometheusMetricsService } from '../prometheus-metrics.service'; import type express from 'express'; import type { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; +import { mockInstance } from '@test/mocking'; +import { GlobalConfig } from '@n8n/config'; const mockMiddleware = ( _req: express.Request, @@ -16,13 +18,27 @@ jest.mock('prom-client'); jest.mock('express-prom-bundle', () => jest.fn(() => mockMiddleware)); describe('PrometheusMetricsService', () => { - beforeEach(() => { - config.load(config.default); + const globalConfig = mockInstance(GlobalConfig, { + endpoints: { + metrics: { + prefix: 'n8n_', + includeDefaultMetrics: true, + includeApiEndpoints: true, + includeCacheMetrics: true, + includeMessageEventBusMetrics: true, + includeCredentialTypeLabel: false, + includeNodeTypeLabel: false, + includeWorkflowIdLabel: false, + includeApiPathLabel: true, + includeApiMethodLabel: true, + includeApiStatusCodeLabel: true, + }, + }, }); describe('init', () => { it('should set up `n8n_version_info`', async () => { - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); await service.init(mock()); @@ -34,7 +50,7 @@ describe('PrometheusMetricsService', () => { }); it('should set up default metrics collection with `prom-client`', async () => { - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); await service.init(mock()); @@ -43,7 +59,7 @@ describe('PrometheusMetricsService', () => { it('should set up `n8n_cache_hits_total`', async () => { config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); await service.init(mock()); @@ -58,7 +74,7 @@ describe('PrometheusMetricsService', () => { it('should set up `n8n_cache_misses_total`', async () => { config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); await service.init(mock()); @@ -73,7 +89,7 @@ describe('PrometheusMetricsService', () => { it('should set up `n8n_cache_updates_total`', async () => { config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); await service.init(mock()); @@ -91,7 +107,7 @@ describe('PrometheusMetricsService', () => { config.set('endpoints.metrics.includeApiPathLabel', true); config.set('endpoints.metrics.includeApiMethodLabel', true); config.set('endpoints.metrics.includeApiStatusCodeLabel', true); - const service = new PrometheusMetricsService(mock(), mock()); + const service = new PrometheusMetricsService(mock(), mock(), globalConfig); const app = mock(); @@ -122,7 +138,7 @@ describe('PrometheusMetricsService', () => { it('should set up event bus metrics', async () => { const eventBus = mock(); - const service = new PrometheusMetricsService(mock(), eventBus); + const service = new PrometheusMetricsService(mock(), eventBus, globalConfig); await service.init(mock()); diff --git a/packages/cli/src/metrics/prometheus-metrics.service.ts b/packages/cli/src/metrics/prometheus-metrics.service.ts index b2d38424bc..1444f6f694 100644 --- a/packages/cli/src/metrics/prometheus-metrics.service.ts +++ b/packages/cli/src/metrics/prometheus-metrics.service.ts @@ -1,4 +1,3 @@ -import config from '@/config'; import { N8N_VERSION } from '@/constants'; import type express from 'express'; import promBundle from 'express-prom-bundle'; @@ -11,32 +10,34 @@ import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; import { EventMessageTypeNames } from 'n8n-workflow'; import type { EventMessageTypes } from '@/eventbus'; import type { Includes, MetricCategory, MetricLabel } from './types'; +import { GlobalConfig } from '@n8n/config'; @Service() export class PrometheusMetricsService { constructor( private readonly cacheService: CacheService, private readonly eventBus: MessageEventBus, + private readonly globalConfig: GlobalConfig, ) {} private readonly counters: { [key: string]: Counter | null } = {}; - private readonly prefix = config.getEnv('endpoints.metrics.prefix'); + private readonly prefix = this.globalConfig.endpoints.metrics.prefix; private readonly includes: Includes = { metrics: { - default: config.getEnv('endpoints.metrics.includeDefaultMetrics'), - routes: config.getEnv('endpoints.metrics.includeApiEndpoints'), - cache: config.getEnv('endpoints.metrics.includeCacheMetrics'), - logs: config.getEnv('endpoints.metrics.includeMessageEventBusMetrics'), + default: this.globalConfig.endpoints.metrics.includeDefaultMetrics, + routes: this.globalConfig.endpoints.metrics.includeApiEndpoints, + cache: this.globalConfig.endpoints.metrics.includeCacheMetrics, + logs: this.globalConfig.endpoints.metrics.includeMessageEventBusMetrics, }, labels: { - credentialsType: config.getEnv('endpoints.metrics.includeCredentialTypeLabel'), - nodeType: config.getEnv('endpoints.metrics.includeNodeTypeLabel'), - workflowId: config.getEnv('endpoints.metrics.includeWorkflowIdLabel'), - apiPath: config.getEnv('endpoints.metrics.includeApiPathLabel'), - apiMethod: config.getEnv('endpoints.metrics.includeApiMethodLabel'), - apiStatusCode: config.getEnv('endpoints.metrics.includeApiStatusCodeLabel'), + credentialsType: this.globalConfig.endpoints.metrics.includeCredentialTypeLabel, + nodeType: this.globalConfig.endpoints.metrics.includeNodeTypeLabel, + workflowId: this.globalConfig.endpoints.metrics.includeWorkflowIdLabel, + apiPath: this.globalConfig.endpoints.metrics.includeApiPathLabel, + apiMethod: this.globalConfig.endpoints.metrics.includeApiMethodLabel, + apiStatusCode: this.globalConfig.endpoints.metrics.includeApiStatusCodeLabel, }, }; diff --git a/packages/cli/src/Mfa/constants.ts b/packages/cli/src/mfa/constants.ts similarity index 100% rename from packages/cli/src/Mfa/constants.ts rename to packages/cli/src/mfa/constants.ts diff --git a/packages/cli/src/Mfa/helpers.ts b/packages/cli/src/mfa/helpers.ts similarity index 100% rename from packages/cli/src/Mfa/helpers.ts rename to packages/cli/src/mfa/helpers.ts diff --git a/packages/cli/src/Mfa/mfa.service.ts b/packages/cli/src/mfa/mfa.service.ts similarity index 85% rename from packages/cli/src/Mfa/mfa.service.ts rename to packages/cli/src/mfa/mfa.service.ts index 91270cf4ab..aa6fddb5a9 100644 --- a/packages/cli/src/Mfa/mfa.service.ts +++ b/packages/cli/src/mfa/mfa.service.ts @@ -3,6 +3,7 @@ import { Service } from 'typedi'; import { Cipher } from 'n8n-core'; import { AuthUserRepository } from '@db/repositories/authUser.repository'; import { TOTPService } from './totp.service'; +import { InvalidMfaCodeError } from '@/errors/response-errors/invalid-mfa-code.error'; @Service() export class MfaService { @@ -60,7 +61,9 @@ export class MfaService { if (mfaToken) { const decryptedSecret = this.cipher.decrypt(user.mfaSecret!); return this.totp.verifySecret({ secret: decryptedSecret, token: mfaToken }); - } else if (mfaRecoveryCode) { + } + + if (mfaRecoveryCode) { const validCodes = user.mfaRecoveryCodes.map((code) => this.cipher.decrypt(code)); const index = validCodes.indexOf(mfaRecoveryCode); if (index === -1) return false; @@ -70,6 +73,7 @@ export class MfaService { await this.authUserRepository.save(user); return true; } + return false; } @@ -79,11 +83,16 @@ export class MfaService { return await this.authUserRepository.save(user); } - async disableMfa(userId: string) { - const user = await this.authUserRepository.findOneByOrFail({ id: userId }); - user.mfaEnabled = false; - user.mfaSecret = null; - user.mfaRecoveryCodes = []; - return await this.authUserRepository.save(user); + async disableMfa(userId: string, mfaToken: string) { + const isValidToken = await this.validateMfa(userId, mfaToken, undefined); + if (!isValidToken) { + throw new InvalidMfaCodeError(); + } + + await this.authUserRepository.update(userId, { + mfaEnabled: false, + mfaSecret: null, + mfaRecoveryCodes: [], + }); } } diff --git a/packages/cli/src/Mfa/totp.service.ts b/packages/cli/src/mfa/totp.service.ts similarity index 100% rename from packages/cli/src/Mfa/totp.service.ts rename to packages/cli/src/mfa/totp.service.ts diff --git a/packages/cli/src/middlewares/bodyParser.ts b/packages/cli/src/middlewares/body-parser.ts similarity index 94% rename from packages/cli/src/middlewares/bodyParser.ts rename to packages/cli/src/middlewares/body-parser.ts index d48bf593cc..5efe388e6f 100644 --- a/packages/cli/src/middlewares/bodyParser.ts +++ b/packages/cli/src/middlewares/body-parser.ts @@ -6,8 +6,9 @@ import { parse as parseQueryString } from 'querystring'; import { Parser as XmlParser } from 'xml2js'; import { parseIncomingMessage } from 'n8n-core'; import { jsonParse } from 'n8n-workflow'; -import config from '@/config'; import { UnprocessableRequestError } from '@/errors/response-errors/unprocessable.error'; +import { GlobalConfig } from '@n8n/config'; +import Container from 'typedi'; const xmlParser = new XmlParser({ async: true, @@ -16,7 +17,7 @@ const xmlParser = new XmlParser({ explicitArray: false, // Only put properties in array if length > 1 }); -const payloadSizeMax = config.getEnv('endpoints.payloadSizeMax'); +const payloadSizeMax = Container.get(GlobalConfig).endpoints.payloadSizeMax; export const rawBodyReader: RequestHandler = async (req, _res, next) => { parseIncomingMessage(req); diff --git a/packages/cli/src/middlewares/index.ts b/packages/cli/src/middlewares/index.ts index 75ebae01c6..07ca0b2a36 100644 --- a/packages/cli/src/middlewares/index.ts +++ b/packages/cli/src/middlewares/index.ts @@ -1,3 +1,3 @@ -export * from './bodyParser'; +export * from './body-parser'; export * from './cors'; -export * from './listQuery'; +export * from './list-query'; diff --git a/packages/cli/test/unit/middleware/listQuery.test.ts b/packages/cli/src/middlewares/list-query/__tests__/listQuery.test.ts similarity index 96% rename from packages/cli/test/unit/middleware/listQuery.test.ts rename to packages/cli/src/middlewares/list-query/__tests__/listQuery.test.ts index 6da542de38..b6ee124dfa 100644 --- a/packages/cli/test/unit/middleware/listQuery.test.ts +++ b/packages/cli/src/middlewares/list-query/__tests__/listQuery.test.ts @@ -1,7 +1,7 @@ -import { filterListQueryMiddleware } from '@/middlewares/listQuery/filter'; -import { selectListQueryMiddleware } from '@/middlewares/listQuery/select'; -import { paginationListQueryMiddleware } from '@/middlewares/listQuery/pagination'; -import * as ResponseHelper from '@/ResponseHelper'; +import { filterListQueryMiddleware } from '@/middlewares/list-query/filter'; +import { selectListQueryMiddleware } from '@/middlewares/list-query/select'; +import { paginationListQueryMiddleware } from '@/middlewares/list-query/pagination'; +import * as ResponseHelper from '@/response-helper'; import type { ListQuery } from '@/requests'; import type { Response, NextFunction } from 'express'; diff --git a/packages/cli/src/middlewares/listQuery/dtos/base.filter.dto.ts b/packages/cli/src/middlewares/list-query/dtos/base.filter.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/base.filter.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/base.filter.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/base.select.dto.ts b/packages/cli/src/middlewares/list-query/dtos/base.select.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/base.select.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/base.select.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/credentials.filter.dto.ts b/packages/cli/src/middlewares/list-query/dtos/credentials.filter.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/credentials.filter.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/credentials.filter.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/credentials.select.dto.ts b/packages/cli/src/middlewares/list-query/dtos/credentials.select.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/credentials.select.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/credentials.select.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/pagination.dto.ts b/packages/cli/src/middlewares/list-query/dtos/pagination.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/pagination.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/pagination.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/user.filter.dto.ts b/packages/cli/src/middlewares/list-query/dtos/user.filter.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/user.filter.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/user.filter.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/user.select.dto.ts b/packages/cli/src/middlewares/list-query/dtos/user.select.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/user.select.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/user.select.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/workflow.filter.dto.ts b/packages/cli/src/middlewares/list-query/dtos/workflow.filter.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/workflow.filter.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/workflow.filter.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/dtos/workflow.select.dto.ts b/packages/cli/src/middlewares/list-query/dtos/workflow.select.dto.ts similarity index 100% rename from packages/cli/src/middlewares/listQuery/dtos/workflow.select.dto.ts rename to packages/cli/src/middlewares/list-query/dtos/workflow.select.dto.ts diff --git a/packages/cli/src/middlewares/listQuery/filter.ts b/packages/cli/src/middlewares/list-query/filter.ts similarity index 95% rename from packages/cli/src/middlewares/listQuery/filter.ts rename to packages/cli/src/middlewares/list-query/filter.ts index 9db1d85cd8..b3216308ea 100644 --- a/packages/cli/src/middlewares/listQuery/filter.ts +++ b/packages/cli/src/middlewares/list-query/filter.ts @@ -1,4 +1,4 @@ -import * as ResponseHelper from '@/ResponseHelper'; +import * as ResponseHelper from '@/response-helper'; import { WorkflowFilter } from './dtos/workflow.filter.dto'; import { CredentialsFilter } from './dtos/credentials.filter.dto'; import { UserFilter } from './dtos/user.filter.dto'; diff --git a/packages/cli/src/middlewares/list-query/index.ts b/packages/cli/src/middlewares/list-query/index.ts new file mode 100644 index 0000000000..524dcb268e --- /dev/null +++ b/packages/cli/src/middlewares/list-query/index.ts @@ -0,0 +1,17 @@ +import { filterListQueryMiddleware } from './filter'; +import { selectListQueryMiddleware } from './select'; +import { paginationListQueryMiddleware } from './pagination'; +import type { ListQuery } from '@/requests'; +import type { NextFunction, Response } from 'express'; + +export type ListQueryMiddleware = ( + req: ListQuery.Request, + res: Response, + next: NextFunction, +) => void; + +export const listQueryMiddleware: ListQueryMiddleware[] = [ + filterListQueryMiddleware, + selectListQueryMiddleware, + paginationListQueryMiddleware, +]; diff --git a/packages/cli/src/middlewares/listQuery/pagination.ts b/packages/cli/src/middlewares/list-query/pagination.ts similarity index 93% rename from packages/cli/src/middlewares/listQuery/pagination.ts rename to packages/cli/src/middlewares/list-query/pagination.ts index cb101af8c1..e9595a24c4 100644 --- a/packages/cli/src/middlewares/listQuery/pagination.ts +++ b/packages/cli/src/middlewares/list-query/pagination.ts @@ -1,5 +1,5 @@ import { toError } from '@/utils'; -import * as ResponseHelper from '@/ResponseHelper'; +import * as ResponseHelper from '@/response-helper'; import { Pagination } from './dtos/pagination.dto'; import type { ListQuery } from '@/requests'; import type { RequestHandler } from 'express'; diff --git a/packages/cli/src/middlewares/listQuery/select.ts b/packages/cli/src/middlewares/list-query/select.ts similarity index 95% rename from packages/cli/src/middlewares/listQuery/select.ts rename to packages/cli/src/middlewares/list-query/select.ts index 6511410a8c..35e6f17cd9 100644 --- a/packages/cli/src/middlewares/listQuery/select.ts +++ b/packages/cli/src/middlewares/list-query/select.ts @@ -1,7 +1,7 @@ import { WorkflowSelect } from './dtos/workflow.select.dto'; import { UserSelect } from './dtos/user.select.dto'; import { CredentialsSelect } from './dtos/credentials.select.dto'; -import * as ResponseHelper from '@/ResponseHelper'; +import * as ResponseHelper from '@/response-helper'; import { toError } from '@/utils'; import type { RequestHandler } from 'express'; diff --git a/packages/cli/src/middlewares/listQuery/index.ts b/packages/cli/src/middlewares/listQuery/index.ts deleted file mode 100644 index 0c8e2c7427..0000000000 --- a/packages/cli/src/middlewares/listQuery/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { filterListQueryMiddleware } from './filter'; -import { selectListQueryMiddleware } from './select'; -import { paginationListQueryMiddleware } from './pagination'; - -export const listQueryMiddleware = [ - filterListQueryMiddleware, - selectListQueryMiddleware, - paginationListQueryMiddleware, -]; diff --git a/packages/cli/src/NodeTypes.ts b/packages/cli/src/node-types.ts similarity index 98% rename from packages/cli/src/NodeTypes.ts rename to packages/cli/src/node-types.ts index 4d9e773167..dc8ea2860c 100644 --- a/packages/cli/src/NodeTypes.ts +++ b/packages/cli/src/node-types.ts @@ -8,7 +8,7 @@ import type { } from 'n8n-workflow'; import { ApplicationError, NodeHelpers } from 'n8n-workflow'; import { Service } from 'typedi'; -import { LoadNodesAndCredentials } from './LoadNodesAndCredentials'; +import { LoadNodesAndCredentials } from './load-nodes-and-credentials'; import { join, dirname } from 'path'; import { readdir } from 'fs/promises'; import type { Dirent } from 'fs'; diff --git a/packages/cli/src/permissions/checkAccess.ts b/packages/cli/src/permissions/check-access.ts similarity index 100% rename from packages/cli/src/permissions/checkAccess.ts rename to packages/cli/src/permissions/check-access.ts diff --git a/packages/cli/src/permissions/project-roles.ts b/packages/cli/src/permissions/project-roles.ts index 96e4dfff8a..d05507c47c 100644 --- a/packages/cli/src/permissions/project-roles.ts +++ b/packages/cli/src/permissions/project-roles.ts @@ -20,6 +20,7 @@ export const REGULAR_PROJECT_ADMIN_SCOPES: Scope[] = [ 'credential:delete', 'credential:list', 'credential:move', + 'credential:share', 'project:list', 'project:read', 'project:update', diff --git a/packages/cli/test/unit/PostHog.test.ts b/packages/cli/src/posthog/__tests__/PostHog.test.ts similarity index 97% rename from packages/cli/test/unit/PostHog.test.ts rename to packages/cli/src/posthog/__tests__/PostHog.test.ts index 5798c0cce2..f1604f7253 100644 --- a/packages/cli/test/unit/PostHog.test.ts +++ b/packages/cli/src/posthog/__tests__/PostHog.test.ts @@ -2,7 +2,7 @@ import { PostHog } from 'posthog-node'; import { InstanceSettings } from 'n8n-core'; import { PostHogClient } from '@/posthog'; import config from '@/config'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; jest.mock('posthog-node'); diff --git a/packages/cli/test/unit/push/index.test.ts b/packages/cli/src/push/__tests__/index.test.ts similarity index 96% rename from packages/cli/test/unit/push/index.test.ts rename to packages/cli/src/push/__tests__/index.test.ts index 1736693509..a61496b0c9 100644 --- a/packages/cli/test/unit/push/index.test.ts +++ b/packages/cli/src/push/__tests__/index.test.ts @@ -9,7 +9,7 @@ import { WebSocketPush } from '@/push/websocket.push'; import type { WebSocketPushRequest, SSEPushRequest } from '@/push/types'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; jest.unmock('@/push'); diff --git a/packages/cli/test/unit/push/websocket.push.test.ts b/packages/cli/src/push/__tests__/websocket.push.test.ts similarity index 97% rename from packages/cli/test/unit/push/websocket.push.test.ts rename to packages/cli/src/push/__tests__/websocket.push.test.ts index 7531e43776..81d4fda0fb 100644 --- a/packages/cli/test/unit/push/websocket.push.test.ts +++ b/packages/cli/src/push/__tests__/websocket.push.test.ts @@ -3,10 +3,10 @@ import { EventEmitter } from 'events'; import type WebSocket from 'ws'; import { WebSocketPush } from '@/push/websocket.push'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { PushDataExecutionRecovered } from '@/Interfaces'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; jest.useFakeTimers(); diff --git a/packages/cli/src/push/abstract.push.ts b/packages/cli/src/push/abstract.push.ts index e9ada3c4bb..67beb7f30a 100644 --- a/packages/cli/src/push/abstract.push.ts +++ b/packages/cli/src/push/abstract.push.ts @@ -1,6 +1,6 @@ import { assert, jsonStringify } from 'n8n-workflow'; import type { IPushDataType } from '@/Interfaces'; -import type { Logger } from '@/Logger'; +import type { Logger } from '@/logger'; /** * Abstract class for two-way push communication. diff --git a/packages/cli/src/push/index.ts b/packages/cli/src/push/index.ts index 647ab40acf..2336862eb7 100644 --- a/packages/cli/src/push/index.ts +++ b/packages/cli/src/push/index.ts @@ -1,4 +1,3 @@ -import { EventEmitter } from 'events'; import { ServerResponse } from 'http'; import type { Server } from 'http'; import type { Socket } from 'net'; @@ -8,7 +7,7 @@ import { parse as parseUrl } from 'url'; import { Container, Service } from 'typedi'; import config from '@/config'; -import { OnShutdown } from '@/decorators/OnShutdown'; +import { OnShutdown } from '@/decorators/on-shutdown'; import { AuthService } from '@/auth/auth.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import type { IPushDataType } from '@/Interfaces'; @@ -17,6 +16,11 @@ import { OrchestrationService } from '@/services/orchestration.service'; import { SSEPush } from './sse.push'; import { WebSocketPush } from './websocket.push'; import type { PushResponse, SSEPushRequest, WebSocketPushRequest } from './types'; +import { TypedEmitter } from '@/TypedEmitter'; + +type PushEvents = { + editorUiConnected: string; +}; const useWebSockets = config.getEnv('push.backend') === 'websocket'; @@ -28,7 +32,7 @@ const useWebSockets = config.getEnv('push.backend') === 'websocket'; * @emits message when a message is received from a client */ @Service() -export class Push extends EventEmitter { +export class Push extends TypedEmitter { private backend = useWebSockets ? Container.get(WebSocketPush) : Container.get(SSEPush); constructor(private readonly orchestrationService: OrchestrationService) { @@ -37,7 +41,6 @@ export class Push extends EventEmitter { handleRequest(req: SSEPushRequest | WebSocketPushRequest, res: PushResponse) { const { - user, ws, query: { pushRef }, } = req; diff --git a/packages/cli/src/push/sse.push.ts b/packages/cli/src/push/sse.push.ts index 8367c74507..38779ed730 100644 --- a/packages/cli/src/push/sse.push.ts +++ b/packages/cli/src/push/sse.push.ts @@ -1,7 +1,7 @@ import SSEChannel from 'sse-channel'; import { Service } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { AbstractPush } from './abstract.push'; import type { PushRequest, PushResponse } from './types'; diff --git a/packages/cli/src/push/websocket.push.ts b/packages/cli/src/push/websocket.push.ts index 04815038ce..733eebdc60 100644 --- a/packages/cli/src/push/websocket.push.ts +++ b/packages/cli/src/push/websocket.push.ts @@ -1,6 +1,6 @@ import type WebSocket from 'ws'; import { Service } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { AbstractPush } from './abstract.push'; function heartbeat(this: WebSocket) { diff --git a/packages/cli/src/requests.ts b/packages/cli/src/requests.ts index 1ff6537257..bae7990c1a 100644 --- a/packages/cli/src/requests.ts +++ b/packages/cli/src/requests.ts @@ -8,12 +8,13 @@ import type { INodeCredentials, INodeParameters, INodeTypeNameVersion, + IPersonalizationSurveyAnswersV4, IUser, } from 'n8n-workflow'; import { Expose } from 'class-transformer'; import { IsBoolean, IsEmail, IsIn, IsOptional, IsString, Length } from 'class-validator'; -import { NoXss } from '@db/utils/customValidators'; +import { NoXss } from '@/validators/no-xss.validator'; import type { PublicUser, SecretsProvider, SecretsProviderState } from '@/Interfaces'; import { AssignableRole } from '@db/entities/User'; import type { GlobalRole, User } from '@db/entities/User'; @@ -25,6 +26,8 @@ import type { Project, ProjectType } from '@db/entities/Project'; import type { ProjectRole } from './databases/entities/ProjectRelation'; import type { Scope } from '@n8n/permissions'; import type { ScopesField } from './services/role.service'; +import type { AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; +import { NoUrl } from '@/validators/no-url.validator'; export class UserUpdatePayload implements Pick { @Expose() @@ -33,15 +36,22 @@ export class UserUpdatePayload implements Pick = APIRequest & { - user: never; -}; +> = APIRequest; export type AuthenticatedRequest< RouteParams = {}, @@ -226,9 +234,9 @@ export declare namespace MeRequest { export type Password = AuthenticatedRequest< {}, {}, - { currentPassword: string; newPassword: string; token?: string } + { currentPassword: string; newPassword: string; mfaCode?: string } >; - export type SurveyAnswers = AuthenticatedRequest<{}, {}, Record | {}>; + export type SurveyAnswers = AuthenticatedRequest<{}, {}, IPersonalizationSurveyAnswersV4>; } export interface UserSetupPayload { @@ -308,7 +316,7 @@ export declare namespace UserRequest { { id: string; email: string; identifier: string }, {}, {}, - { limit?: number; offset?: number; cursor?: string; includeRole?: boolean } + { limit?: number; offset?: number; cursor?: string; includeRole?: boolean; projectId?: string } >; export type PasswordResetLink = AuthenticatedRequest<{ id: string }, {}, {}, {}>; @@ -355,6 +363,7 @@ export type LoginRequest = AuthlessRequest< export declare namespace MFA { type Verify = AuthenticatedRequest<{}, {}, { token: string }, {}>; type Activate = AuthenticatedRequest<{}, {}, { token: string }, {}>; + type Disable = AuthenticatedRequest<{}, {}, { token: string }, {}>; type Config = AuthenticatedRequest<{}, {}, { login: { enabled: boolean } }, {}>; type ValidateRecoveryCode = AuthenticatedRequest< {}, @@ -371,7 +380,7 @@ export declare namespace MFA { export declare namespace OAuthRequest { namespace OAuth1Credential { type Auth = AuthenticatedRequest<{}, {}, {}, { id: string }>; - type Callback = AuthenticatedRequest< + type Callback = AuthlessRequest< {}, {}, {}, @@ -383,7 +392,7 @@ export declare namespace OAuthRequest { namespace OAuth2Credential { type Auth = AuthenticatedRequest<{}, {}, {}, { id: string }>; - type Callback = AuthenticatedRequest<{}, {}, {}, { code: string; state: string }>; + type Callback = AuthlessRequest<{}, {}, {}, { code: string; state: string }>; } } @@ -420,6 +429,12 @@ export declare namespace DynamicNodeParametersRequest { type ResourceMapperFields = BaseRequest<{ methodName: string; }>; + + /** POST /dynamic-node-parameters/action-result */ + type ActionResult = BaseRequest<{ + handler: string; + payload: IDataObject | string | undefined; + }>; } // ---------------------------------- @@ -602,3 +617,14 @@ export declare namespace NpsSurveyRequest { // once some schema validation is added type NpsSurveyUpdate = AuthenticatedRequest<{}, {}, unknown>; } + +// ---------------------------------- +// /ai-assistant +// ---------------------------------- + +export declare namespace AiAssistantRequest { + type Chat = AuthenticatedRequest<{}, {}, AiAssistantSDK.ChatRequestPayload>; + + type SuggestionPayload = { sessionId: string; suggestionId: string }; + type ApplySuggestion = AuthenticatedRequest<{}, {}, SuggestionPayload>; +} diff --git a/packages/cli/src/ResponseHelper.ts b/packages/cli/src/response-helper.ts similarity index 99% rename from packages/cli/src/ResponseHelper.ts rename to packages/cli/src/response-helper.ts index 3d172521d9..aebef56f0c 100644 --- a/packages/cli/src/ResponseHelper.ts +++ b/packages/cli/src/response-helper.ts @@ -11,7 +11,7 @@ import { Readable } from 'node:stream'; import { inDevelopment } from '@/constants'; import { ResponseError } from './errors/response-errors/abstract/response.error'; import Container from 'typedi'; -import { Logger } from './Logger'; +import { Logger } from './logger'; export function sendSuccessResponse( res: Response, diff --git a/packages/cli/src/scaling/__tests__/scaling.service.test.ts b/packages/cli/src/scaling/__tests__/scaling.service.test.ts new file mode 100644 index 0000000000..adbf5ebde2 --- /dev/null +++ b/packages/cli/src/scaling/__tests__/scaling.service.test.ts @@ -0,0 +1,358 @@ +import { mock } from 'jest-mock-extended'; +import { ScalingService } from '../scaling.service'; +import { JOB_TYPE_NAME, QUEUE_NAME } from '../constants'; +import config from '@/config'; +import * as BullModule from 'bull'; +import type { Job, JobData, JobOptions, JobQueue } from '../types'; +import { ApplicationError } from 'n8n-workflow'; +import { mockInstance } from '@test/mocking'; +import { GlobalConfig } from '@n8n/config'; +import { InstanceSettings } from 'n8n-core'; +import type { OrchestrationService } from '@/services/orchestration.service'; +import Container from 'typedi'; +import type { JobProcessor } from '../job-processor'; + +const queue = mock({ + client: { ping: jest.fn() }, +}); + +jest.mock('bull', () => ({ + __esModule: true, + default: jest.fn(() => queue), +})); + +describe('ScalingService', () => { + const globalConfig = mockInstance(GlobalConfig, { + queue: { + bull: { + prefix: 'bull', + redis: { + clusterNodes: '', + host: 'localhost', + password: '', + port: 6379, + tls: false, + }, + }, + }, + }); + + const instanceSettings = Container.get(InstanceSettings); + const orchestrationService = mock({ isMultiMainSetupEnabled: false }); + const jobProcessor = mock(); + let scalingService: ScalingService; + + beforeEach(() => { + jest.clearAllMocks(); + config.set('generic.instanceType', 'main'); + scalingService = new ScalingService( + mock(), + mock(), + jobProcessor, + globalConfig, + mock(), + instanceSettings, + orchestrationService, + ); + }); + + afterEach(() => { + scalingService.stopQueueRecovery(); + }); + + describe('setupQueue', () => { + it('should set up the queue', async () => { + /** + * Arrange + */ + const { prefix, settings } = globalConfig.queue.bull; + const Bull = jest.mocked(BullModule.default); + + /** + * Act + */ + await scalingService.setupQueue(); + + /** + * Assert + */ + expect(Bull).toHaveBeenCalledWith(QUEUE_NAME, { + prefix, + settings, + createClient: expect.any(Function), + }); + expect(queue.on).toHaveBeenCalledWith('global:progress', expect.any(Function)); + expect(queue.on).toHaveBeenCalledWith('error', expect.any(Function)); + }); + }); + + describe('setupWorker', () => { + it('should set up a worker with concurrency', async () => { + /** + * Arrange + */ + config.set('generic.instanceType', 'worker'); + const scalingService = new ScalingService( + mock(), + mock(), + mock(), + globalConfig, + mock(), + instanceSettings, + orchestrationService, + ); + await scalingService.setupQueue(); + const concurrency = 5; + + /** + * Act + */ + scalingService.setupWorker(concurrency); + + /** + * Assert + */ + expect(queue.process).toHaveBeenCalledWith(JOB_TYPE_NAME, concurrency, expect.any(Function)); + }); + + it('should throw if called on a non-worker instance', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + + /** + * Act and Assert + */ + expect(() => scalingService.setupWorker(5)).toThrow(); + }); + }); + + describe('stop', () => { + it('should pause the queue, check for running jobs, and stop queue recovery', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + jobProcessor.getRunningJobIds.mockReturnValue([]); + const stopQueueRecoverySpy = jest.spyOn(scalingService, 'stopQueueRecovery'); + const getRunningJobsCountSpy = jest.spyOn(scalingService, 'getRunningJobsCount'); + + /** + * Act + */ + await scalingService.stop(); + + /** + * Assert + */ + expect(queue.pause).toHaveBeenCalledWith(true, true); + expect(stopQueueRecoverySpy).toHaveBeenCalled(); + expect(getRunningJobsCountSpy).toHaveBeenCalled(); + }); + }); + + describe('pingQueue', () => { + it('should ping the queue', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + + /** + * Act + */ + await scalingService.pingQueue(); + + /** + * Assert + */ + expect(queue.client.ping).toHaveBeenCalled(); + }); + }); + + describe('addJob', () => { + it('should add a job', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + queue.add.mockResolvedValue(mock({ id: '456' })); + + /** + * Act + */ + const jobData = mock({ executionId: '123' }); + const jobOptions = mock(); + await scalingService.addJob(jobData, jobOptions); + + /** + * Assert + */ + expect(queue.add).toHaveBeenCalledWith(JOB_TYPE_NAME, jobData, jobOptions); + }); + }); + + describe('getJob', () => { + it('should get a job', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + const jobId = '123'; + queue.getJob.mockResolvedValue(mock({ id: jobId })); + + /** + * Act + */ + const job = await scalingService.getJob(jobId); + + /** + * Assert + */ + expect(queue.getJob).toHaveBeenCalledWith(jobId); + expect(job?.id).toBe(jobId); + }); + }); + + describe('findJobsByStatus', () => { + it('should find jobs by status', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + queue.getJobs.mockResolvedValue([mock({ id: '123' })]); + + /** + * Act + */ + const jobs = await scalingService.findJobsByStatus(['active']); + + /** + * Assert + */ + expect(queue.getJobs).toHaveBeenCalledWith(['active']); + expect(jobs).toHaveLength(1); + expect(jobs.at(0)?.id).toBe('123'); + }); + + it('should filter out `null` in Redis response', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + // @ts-expect-error - Untyped but possible Redis response + queue.getJobs.mockResolvedValue([mock(), null]); + + /** + * Act + */ + const jobs = await scalingService.findJobsByStatus(['waiting']); + + /** + * Assert + */ + expect(jobs).toHaveLength(1); + }); + }); + + describe('stopJob', () => { + it('should stop an active job', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + const job = mock({ isActive: jest.fn().mockResolvedValue(true) }); + + /** + * Act + */ + const result = await scalingService.stopJob(job); + + /** + * Assert + */ + expect(job.progress).toHaveBeenCalledWith({ kind: 'abort-job' }); + expect(result).toBe(true); + }); + + it('should stop an inactive job', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + const job = mock({ isActive: jest.fn().mockResolvedValue(false) }); + + /** + * Act + */ + const result = await scalingService.stopJob(job); + + /** + * Assert + */ + expect(job.remove).toHaveBeenCalled(); + expect(result).toBe(true); + }); + + it('should report failure to stop a job', async () => { + /** + * Arrange + */ + await scalingService.setupQueue(); + const job = mock({ + isActive: jest.fn().mockImplementation(() => { + throw new ApplicationError('Something went wrong'); + }), + }); + + /** + * Act + */ + const result = await scalingService.stopJob(job); + + /** + * Assert + */ + expect(result).toBe(false); + }); + }); + + describe('scheduleQueueRecovery', () => { + it('if leader, should schedule queue recovery', async () => { + /** + * Arrange + */ + const scheduleSpy = jest.spyOn(scalingService, 'scheduleQueueRecovery'); + instanceSettings.markAsLeader(); + + /** + * Act + */ + await scalingService.setupQueue(); + + /** + * Assert + */ + expect(scheduleSpy).toHaveBeenCalled(); + }); + + it('if follower, should not schedule queue recovery', async () => { + /** + * Arrange + */ + const scheduleSpy = jest.spyOn(scalingService, 'scheduleQueueRecovery'); + instanceSettings.markAsFollower(); + + /** + * Act + */ + await scalingService.setupQueue(); + + /** + * Assert + */ + expect(scheduleSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/cli/src/scaling/constants.ts b/packages/cli/src/scaling/constants.ts new file mode 100644 index 0000000000..8ef5f716b1 --- /dev/null +++ b/packages/cli/src/scaling/constants.ts @@ -0,0 +1,3 @@ +export const QUEUE_NAME = 'jobs'; + +export const JOB_TYPE_NAME = 'job'; diff --git a/packages/cli/src/scaling/job-processor.ts b/packages/cli/src/scaling/job-processor.ts new file mode 100644 index 0000000000..8618424e35 --- /dev/null +++ b/packages/cli/src/scaling/job-processor.ts @@ -0,0 +1,182 @@ +import { Service } from 'typedi'; +import { BINARY_ENCODING, ApplicationError, Workflow } from 'n8n-workflow'; +import { WorkflowExecute } from 'n8n-core'; +import { Logger } from '@/logger'; +import config from '@/config'; +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import { NodeTypes } from '@/node-types'; +import type { ExecutionStatus, IExecuteResponsePromiseData, IRun } from 'n8n-workflow'; +import type { Job, JobId, JobResult, RunningJob, RunningJobSummary } from './types'; +import type PCancelable from 'p-cancelable'; + +/** + * Responsible for processing jobs from the queue, i.e. running enqueued executions. + */ +@Service() +export class JobProcessor { + private readonly runningJobs: Record = {}; + + constructor( + private readonly logger: Logger, + private readonly executionRepository: ExecutionRepository, + private readonly workflowRepository: WorkflowRepository, + private readonly nodeTypes: NodeTypes, + ) {} + + async processJob(job: Job): Promise { + const { executionId, loadStaticData } = job.data; + + const execution = await this.executionRepository.findSingleExecution(executionId, { + includeData: true, + unflattenData: true, + }); + + if (!execution) { + this.logger.error('[JobProcessor] Failed to find execution data', { executionId }); + throw new ApplicationError('Failed to find execution data. Aborting execution.', { + extra: { executionId }, + }); + } + + const workflowId = execution.workflowData.id; + + this.logger.info(`[JobProcessor] Starting job ${job.id} (execution ${executionId})`); + + await this.executionRepository.updateStatus(executionId, 'running'); + + let { staticData } = execution.workflowData; + + if (loadStaticData) { + const workflowData = await this.workflowRepository.findOne({ + select: ['id', 'staticData'], + where: { id: workflowId }, + }); + + if (workflowData === null) { + this.logger.error('[JobProcessor] Failed to find workflow', { workflowId, executionId }); + throw new ApplicationError('Failed to find workflow', { extra: { workflowId } }); + } + + staticData = workflowData.staticData; + } + + const workflowSettings = execution.workflowData.settings ?? {}; + + let workflowTimeout = workflowSettings.executionTimeout ?? config.getEnv('executions.timeout'); + + let executionTimeoutTimestamp: number | undefined; + + if (workflowTimeout > 0) { + workflowTimeout = Math.min(workflowTimeout, config.getEnv('executions.maxTimeout')); + executionTimeoutTimestamp = Date.now() + workflowTimeout * 1000; + } + + const workflow = new Workflow({ + id: workflowId, + name: execution.workflowData.name, + nodes: execution.workflowData.nodes, + connections: execution.workflowData.connections, + active: execution.workflowData.active, + nodeTypes: this.nodeTypes, + staticData, + settings: execution.workflowData.settings, + }); + + const additionalData = await WorkflowExecuteAdditionalData.getBase( + undefined, + undefined, + executionTimeoutTimestamp, + ); + + additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( + execution.mode, + job.data.executionId, + execution.workflowData, + { retryOf: execution.retryOf as string }, + ); + + additionalData.hooks.hookFunctions.sendResponse = [ + async (response: IExecuteResponsePromiseData): Promise => { + await job.progress({ + kind: 'respond-to-webhook', + executionId, + response: this.encodeWebhookResponse(response), + }); + }, + ]; + + additionalData.executionId = executionId; + + additionalData.setExecutionStatus = (status: ExecutionStatus) => { + // Can't set the status directly in the queued worker, but it will happen in InternalHook.onWorkflowPostExecute + this.logger.debug( + `[JobProcessor] Queued worker execution status for ${executionId} is "${status}"`, + ); + }; + + let workflowExecute: WorkflowExecute; + let workflowRun: PCancelable; + if (execution.data !== undefined) { + workflowExecute = new WorkflowExecute(additionalData, execution.mode, execution.data); + workflowRun = workflowExecute.processRunExecutionData(workflow); + } else { + // Execute all nodes + // Can execute without webhook so go on + workflowExecute = new WorkflowExecute(additionalData, execution.mode); + workflowRun = workflowExecute.run(workflow); + } + + const runningJob: RunningJob = { + run: workflowRun, + executionId, + workflowId: execution.workflowId, + workflowName: execution.workflowData.name, + mode: execution.mode, + startedAt: execution.startedAt, + retryOf: execution.retryOf ?? '', + status: execution.status, + }; + + this.runningJobs[job.id] = runningJob; + + await workflowRun; + + delete this.runningJobs[job.id]; + + this.logger.debug('[JobProcessor] Job finished running', { jobId: job.id, executionId }); + + /** + * @important Do NOT call `workflowExecuteAfter` hook here. + * It is being called from processSuccessExecution() already. + */ + + return { success: true }; + } + + stopJob(jobId: JobId) { + this.runningJobs[jobId]?.run.cancel(); + delete this.runningJobs[jobId]; + } + + getRunningJobIds(): JobId[] { + return Object.keys(this.runningJobs); + } + + getRunningJobsSummary(): RunningJobSummary[] { + return Object.values(this.runningJobs).map(({ run, ...summary }) => summary); + } + + private encodeWebhookResponse( + response: IExecuteResponsePromiseData, + ): IExecuteResponsePromiseData { + if (typeof response === 'object' && Buffer.isBuffer(response.body)) { + response.body = { + '__@N8nEncodedBuffer@__': response.body.toString(BINARY_ENCODING), + }; + } + + return response; + } +} diff --git a/packages/cli/src/scaling/scaling.service.ts b/packages/cli/src/scaling/scaling.service.ts new file mode 100644 index 0000000000..4dbdce1aac --- /dev/null +++ b/packages/cli/src/scaling/scaling.service.ts @@ -0,0 +1,342 @@ +import Container, { Service } from 'typedi'; +import { ApplicationError, BINARY_ENCODING, sleep, jsonStringify } from 'n8n-workflow'; +import { ActiveExecutions } from '@/active-executions'; +import config from '@/config'; +import { Logger } from '@/logger'; +import { MaxStalledCountError } from '@/errors/max-stalled-count.error'; +import { HIGHEST_SHUTDOWN_PRIORITY, Time } from '@/constants'; +import { OnShutdown } from '@/decorators/on-shutdown'; +import { JOB_TYPE_NAME, QUEUE_NAME } from './constants'; +import { JobProcessor } from './job-processor'; +import type { + JobQueue, + Job, + JobData, + JobOptions, + JobMessage, + JobStatus, + JobId, + QueueRecoveryContext, +} from './types'; +import type { IExecuteResponsePromiseData } from 'n8n-workflow'; +import { GlobalConfig } from '@n8n/config'; +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { InstanceSettings } from 'n8n-core'; +import { OrchestrationService } from '@/services/orchestration.service'; + +@Service() +export class ScalingService { + private queue: JobQueue; + + private readonly instanceType = config.getEnv('generic.instanceType'); + + constructor( + private readonly logger: Logger, + private readonly activeExecutions: ActiveExecutions, + private readonly jobProcessor: JobProcessor, + private readonly globalConfig: GlobalConfig, + private readonly executionRepository: ExecutionRepository, + private readonly instanceSettings: InstanceSettings, + private readonly orchestrationService: OrchestrationService, + ) {} + + // #region Lifecycle + + async setupQueue() { + const { default: BullQueue } = await import('bull'); + const { RedisClientService } = await import('@/services/redis/redis-client.service'); + const service = Container.get(RedisClientService); + + const bullPrefix = this.globalConfig.queue.bull.prefix; + const prefix = service.toValidPrefix(bullPrefix); + + this.queue = new BullQueue(QUEUE_NAME, { + prefix, + settings: this.globalConfig.queue.bull.settings, + createClient: (type) => service.createClient({ type: `${type}(bull)` }), + }); + + this.registerListeners(); + + if (this.instanceSettings.isLeader) this.scheduleQueueRecovery(); + + if (this.orchestrationService.isMultiMainSetupEnabled) { + this.orchestrationService.multiMainSetup + .on('leader-takeover', () => this.scheduleQueueRecovery()) + .on('leader-stepdown', () => this.stopQueueRecovery()); + } + + this.logger.debug('[ScalingService] Queue setup completed'); + } + + setupWorker(concurrency: number) { + this.assertWorker(); + + void this.queue.process( + JOB_TYPE_NAME, + concurrency, + async (job: Job) => await this.jobProcessor.processJob(job), + ); + + this.logger.debug('[ScalingService] Worker setup completed'); + } + + @OnShutdown(HIGHEST_SHUTDOWN_PRIORITY) + async stop() { + await this.queue.pause(true, true); + + this.logger.debug('[ScalingService] Queue paused'); + + this.stopQueueRecovery(); + + this.logger.debug('[ScalingService] Queue recovery stopped'); + + let count = 0; + + while (this.getRunningJobsCount() !== 0) { + if (count++ % 4 === 0) { + this.logger.info( + `Waiting for ${this.getRunningJobsCount()} active executions to finish...`, + ); + } + + await sleep(500); + } + } + + async pingQueue() { + await this.queue.client.ping(); + } + + // #endregion + + // #region Jobs + + async addJob(jobData: JobData, jobOptions: JobOptions) { + const { executionId } = jobData; + + const job = await this.queue.add(JOB_TYPE_NAME, jobData, jobOptions); + + this.logger.info(`[ScalingService] Added job ${job.id} (execution ${executionId})`); + + return job; + } + + async getJob(jobId: JobId) { + return await this.queue.getJob(jobId); + } + + async findJobsByStatus(statuses: JobStatus[]) { + const jobs = await this.queue.getJobs(statuses); + + return jobs.filter((job) => job !== null); + } + + async stopJob(job: Job) { + const props = { jobId: job.id, executionId: job.data.executionId }; + + try { + if (await job.isActive()) { + await job.progress({ kind: 'abort-job' }); // being processed by worker + this.logger.debug('[ScalingService] Stopped active job', props); + return true; + } + + await job.remove(); // not yet picked up, or waiting for next pickup (stalled) + this.logger.debug('[ScalingService] Stopped inactive job', props); + return true; + } catch (error: unknown) { + await job.progress({ kind: 'abort-job' }); + this.logger.error('[ScalingService] Failed to stop job', { ...props, error }); + return false; + } + } + + getRunningJobsCount() { + return this.jobProcessor.getRunningJobIds().length; + } + + // #endregion + + // #region Listeners + + private registerListeners() { + this.queue.on('global:progress', (_jobId: JobId, msg: JobMessage) => { + if (msg.kind === 'respond-to-webhook') { + const { executionId, response } = msg; + this.activeExecutions.resolveResponsePromise( + executionId, + this.decodeWebhookResponse(response), + ); + } + }); + + this.queue.on('global:progress', (jobId: JobId, msg: JobMessage) => { + if (msg.kind === 'abort-job') { + this.jobProcessor.stopJob(jobId); + } + }); + + let latestAttemptTs = 0; + let cumulativeTimeoutMs = 0; + + const MAX_TIMEOUT_MS = this.globalConfig.queue.bull.redis.timeoutThreshold; + const RESET_LENGTH_MS = 30_000; + + this.queue.on('error', (error: Error) => { + this.logger.error('[ScalingService] Queue errored', { error }); + + /** + * On Redis connection failure, try to reconnect. On every failed attempt, + * increment a cumulative timeout - if this exceeds a limit, exit the + * process. Reset the cumulative timeout if >30s between retries. + */ + if (error.message.includes('ECONNREFUSED')) { + const nowTs = Date.now(); + if (nowTs - latestAttemptTs > RESET_LENGTH_MS) { + latestAttemptTs = nowTs; + cumulativeTimeoutMs = 0; + } else { + cumulativeTimeoutMs += nowTs - latestAttemptTs; + latestAttemptTs = nowTs; + if (cumulativeTimeoutMs > MAX_TIMEOUT_MS) { + this.logger.error('[ScalingService] Redis unavailable after max timeout'); + this.logger.error('[ScalingService] Exiting process...'); + process.exit(1); + } + } + + this.logger.warn('[ScalingService] Redis unavailable - retrying to connect...'); + return; + } + + if ( + this.instanceType === 'worker' && + error.message.includes('job stalled more than maxStalledCount') + ) { + throw new MaxStalledCountError(error); + } + + /** + * Non-recoverable error on worker start with Redis unavailable. + * Even if Redis recovers, worker will remain unable to process jobs. + */ + if ( + this.instanceType === 'worker' && + error.message.includes('Error initializing Lua scripts') + ) { + this.logger.error('[ScalingService] Fatal error initializing worker', { error }); + this.logger.error('[ScalingService] Exiting process...'); + process.exit(1); + } + + throw error; + }); + } + + // #endregion + + private decodeWebhookResponse( + response: IExecuteResponsePromiseData, + ): IExecuteResponsePromiseData { + if ( + typeof response === 'object' && + typeof response.body === 'object' && + response.body !== null && + '__@N8nEncodedBuffer@__' in response.body && + typeof response.body['__@N8nEncodedBuffer@__'] === 'string' + ) { + response.body = Buffer.from(response.body['__@N8nEncodedBuffer@__'], BINARY_ENCODING); + } + + return response; + } + + private assertWorker() { + if (this.instanceType === 'worker') return; + + throw new ApplicationError('This method must be called on a `worker` instance'); + } + + // #region Queue recovery + + private readonly queueRecoveryContext: QueueRecoveryContext = { + batchSize: config.getEnv('executions.queueRecovery.batchSize'), + waitMs: config.getEnv('executions.queueRecovery.interval') * 60 * 1000, + }; + + scheduleQueueRecovery(waitMs = this.queueRecoveryContext.waitMs) { + this.queueRecoveryContext.timeout = setTimeout(async () => { + try { + const nextWaitMs = await this.recoverFromQueue(); + this.scheduleQueueRecovery(nextWaitMs); + } catch (error) { + this.logger.error('[ScalingService] Failed to recover dangling executions from queue', { + msg: this.toErrorMsg(error), + }); + this.logger.error('[ScalingService] Retrying...'); + + this.scheduleQueueRecovery(); + } + }, waitMs); + + const wait = [this.queueRecoveryContext.waitMs / Time.minutes.toMilliseconds, 'min'].join(' '); + + this.logger.debug(`[ScalingService] Scheduled queue recovery check for next ${wait}`); + } + + stopQueueRecovery() { + clearTimeout(this.queueRecoveryContext.timeout); + } + + /** + * Mark in-progress executions as `crashed` if stored in DB as `new` or `running` + * but absent from the queue. Return time until next recovery cycle. + */ + private async recoverFromQueue() { + const { waitMs, batchSize } = this.queueRecoveryContext; + + const storedIds = await this.executionRepository.getInProgressExecutionIds(batchSize); + + if (storedIds.length === 0) { + this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions'); + return waitMs; + } + + const runningJobs = await this.findJobsByStatus(['active', 'waiting']); + + const queuedIds = new Set(runningJobs.map((job) => job.data.executionId)); + + if (queuedIds.size === 0) { + this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions'); + return waitMs; + } + + const danglingIds = storedIds.filter((id) => !queuedIds.has(id)); + + if (danglingIds.length === 0) { + this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions'); + return waitMs; + } + + await this.executionRepository.markAsCrashed(danglingIds); + + this.logger.info( + '[ScalingService] Completed queue recovery check, recovered dangling executions', + { danglingIds }, + ); + + // if this cycle used up the whole batch size, it is possible for there to be + // dangling executions outside this check, so speed up next cycle + + return storedIds.length >= this.queueRecoveryContext.batchSize ? waitMs / 2 : waitMs; + } + + private toErrorMsg(error: unknown) { + return error instanceof Error + ? error.message + : jsonStringify(error, { replaceCircularRefs: true }); + } + + // #endregion +} diff --git a/packages/cli/src/scaling/types.ts b/packages/cli/src/scaling/types.ts new file mode 100644 index 0000000000..b35d1d109d --- /dev/null +++ b/packages/cli/src/scaling/types.ts @@ -0,0 +1,66 @@ +import type { + ExecutionError, + ExecutionStatus, + IExecuteResponsePromiseData, + IRun, + WorkflowExecuteMode as WorkflowExecutionMode, +} from 'n8n-workflow'; +import type Bull from 'bull'; +import type PCancelable from 'p-cancelable'; + +export type JobQueue = Bull.Queue; + +export type Job = Bull.Job; + +export type JobId = Job['id']; + +export type JobData = { + executionId: string; + loadStaticData: boolean; +}; + +export type JobResult = { + success: boolean; + error?: ExecutionError; +}; + +export type JobStatus = Bull.JobStatus; + +export type JobOptions = Bull.JobOptions; + +/** Message sent by worker to queue or by queue to worker. */ +export type JobMessage = RepondToWebhookMessage | AbortJobMessage; + +export type RepondToWebhookMessage = { + kind: 'respond-to-webhook'; + executionId: string; + response: IExecuteResponsePromiseData; +}; + +export type AbortJobMessage = { + kind: 'abort-job'; +}; + +export type RunningJob = { + executionId: string; + workflowId: string; + workflowName: string; + mode: WorkflowExecutionMode; + startedAt: Date; + retryOf: string; + status: ExecutionStatus; + run: PCancelable; +}; + +export type RunningJobSummary = Omit; + +export type QueueRecoveryContext = { + /** ID of timeout for next scheduled recovery cycle. */ + timeout?: NodeJS.Timeout; + + /** Number of in-progress executions to check per cycle. */ + batchSize: number; + + /** Time (in milliseconds) to wait until the next cycle. */ + waitMs: number; +}; diff --git a/packages/cli/src/SecretsHelpers.ts b/packages/cli/src/secrets-helpers.ts similarity index 91% rename from packages/cli/src/SecretsHelpers.ts rename to packages/cli/src/secrets-helpers.ts index 8555adb36e..082c3ea97d 100644 --- a/packages/cli/src/SecretsHelpers.ts +++ b/packages/cli/src/secrets-helpers.ts @@ -1,6 +1,6 @@ import type { SecretsHelpersBase } from 'n8n-workflow'; import { Service } from 'typedi'; -import { ExternalSecretsManager } from './ExternalSecrets/ExternalSecretsManager.ee'; +import { ExternalSecretsManager } from './external-secrets/external-secrets-manager.ee'; @Service() export class SecretsHelper implements SecretsHelpersBase { diff --git a/packages/cli/src/security-audit/risk-reporters/CredentialsRiskReporter.ts b/packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts similarity index 100% rename from packages/cli/src/security-audit/risk-reporters/CredentialsRiskReporter.ts rename to packages/cli/src/security-audit/risk-reporters/credentials-risk-reporter.ts diff --git a/packages/cli/src/security-audit/risk-reporters/DatabaseRiskReporter.ts b/packages/cli/src/security-audit/risk-reporters/database-risk-reporter.ts similarity index 100% rename from packages/cli/src/security-audit/risk-reporters/DatabaseRiskReporter.ts rename to packages/cli/src/security-audit/risk-reporters/database-risk-reporter.ts diff --git a/packages/cli/src/security-audit/risk-reporters/FilesystemRiskReporter.ts b/packages/cli/src/security-audit/risk-reporters/filesystem-risk-reporter.ts similarity index 100% rename from packages/cli/src/security-audit/risk-reporters/FilesystemRiskReporter.ts rename to packages/cli/src/security-audit/risk-reporters/filesystem-risk-reporter.ts diff --git a/packages/cli/src/security-audit/risk-reporters/InstanceRiskReporter.ts b/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts similarity index 99% rename from packages/cli/src/security-audit/risk-reporters/InstanceRiskReporter.ts rename to packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts index 16e53b400c..323d1173f0 100644 --- a/packages/cli/src/security-audit/risk-reporters/InstanceRiskReporter.ts +++ b/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts @@ -14,7 +14,7 @@ import { getN8nPackageJson, inDevelopment } from '@/constants'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import type { RiskReporter, Risk, n8n } from '@/security-audit/types'; import { isApiEnabled } from '@/PublicApi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { GlobalConfig } from '@n8n/config'; @Service() diff --git a/packages/cli/src/security-audit/risk-reporters/NodesRiskReporter.ts b/packages/cli/src/security-audit/risk-reporters/nodes-risk-reporter.ts similarity index 98% rename from packages/cli/src/security-audit/risk-reporters/NodesRiskReporter.ts rename to packages/cli/src/security-audit/risk-reporters/nodes-risk-reporter.ts index 55fe0a8c0c..9bd7b58466 100644 --- a/packages/cli/src/security-audit/risk-reporters/NodesRiskReporter.ts +++ b/packages/cli/src/security-audit/risk-reporters/nodes-risk-reporter.ts @@ -1,7 +1,7 @@ import * as path from 'path'; import glob from 'fast-glob'; import { Service } from 'typedi'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { getNodeTypes } from '@/security-audit/utils'; import { OFFICIAL_RISKY_NODE_TYPES, diff --git a/packages/cli/src/security-audit/SecurityAudit.service.ts b/packages/cli/src/security-audit/security-audit.service.ts similarity index 83% rename from packages/cli/src/security-audit/SecurityAudit.service.ts rename to packages/cli/src/security-audit/security-audit.service.ts index afec146065..3c27f647bc 100644 --- a/packages/cli/src/security-audit/SecurityAudit.service.ts +++ b/packages/cli/src/security-audit/security-audit.service.ts @@ -51,8 +51,16 @@ export class SecurityAuditService { for (const category of categories) { const className = category.charAt(0).toUpperCase() + category.slice(1) + 'RiskReporter'; + const toFilename: Record = { + CredentialsRiskReporter: 'credentials-risk-reporter', + DatabaseRiskReporter: 'database-risk-reporter', + FilesystemRiskReporter: 'filesystem-risk-reporter', + InstanceRiskReporter: 'instance-risk-reporter', + NodesRiskReporter: 'nodes-risk-reporter', + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const RiskReporterModule = await import(`./risk-reporters/${className}`); + const RiskReporterModule = await import(`./risk-reporters/${toFilename[className]}`); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const RiskReporterClass = RiskReporterModule[className] as { new (): RiskReporter }; diff --git a/packages/cli/src/Server.ts b/packages/cli/src/server.ts similarity index 88% rename from packages/cli/src/Server.ts rename to packages/cli/src/server.ts index a31a87f91d..30de80295a 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/server.ts @@ -6,7 +6,6 @@ import { promisify } from 'util'; import cookieParser from 'cookie-parser'; import express from 'express'; import helmet from 'helmet'; -import { GlobalConfig } from '@n8n/config'; import { InstanceSettings } from 'n8n-core'; import type { IN8nUISettings } from 'n8n-workflow'; @@ -25,24 +24,24 @@ import type { APIRequest } from '@/requests'; import { ControllerRegistry } from '@/decorators'; import { isApiEnabled, loadPublicApiVersions } from '@/PublicApi'; import type { ICredentialsOverwrite } from '@/Interfaces'; -import { CredentialsOverwrites } from '@/CredentialsOverwrites'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; -import * as ResponseHelper from '@/ResponseHelper'; +import { CredentialsOverwrites } from '@/credentials-overwrites'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import * as ResponseHelper from '@/response-helper'; import { setupPushServer, setupPushHandler } from '@/push'; -import { isLdapEnabled } from '@/Ldap/helpers.ee'; -import { AbstractServer } from '@/AbstractServer'; +import { isLdapEnabled } from '@/ldap/helpers.ee'; +import { AbstractServer } from '@/abstract-server'; import { PostHogClient } from '@/posthog'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { InternalHooks } from '@/InternalHooks'; -import { handleMfaDisable, isMfaFeatureEnabled } from '@/Mfa/helpers'; +import { handleMfaDisable, isMfaFeatureEnabled } from '@/mfa/helpers'; import type { FrontendService } from '@/services/frontend.service'; import { OrchestrationService } from '@/services/orchestration.service'; -import { AuditEventRelay } from './eventbus/audit-event-relay.service'; +import { LogStreamingEventRelay } from '@/events/log-streaming-event-relay'; import '@/controllers/activeWorkflows.controller'; import '@/controllers/auth.controller'; import '@/controllers/binaryData.controller'; import '@/controllers/curl.controller'; +import '@/controllers/aiAssistant.controller'; import '@/controllers/dynamicNodeParameters.controller'; import '@/controllers/invitation.controller'; import '@/controllers/me.controller'; @@ -62,10 +61,11 @@ import '@/controllers/workflowStatistics.controller'; import '@/credentials/credentials.controller'; import '@/eventbus/eventBus.controller'; import '@/executions/executions.controller'; -import '@/ExternalSecrets/ExternalSecrets.controller.ee'; +import '@/external-secrets/external-secrets.controller.ee'; import '@/license/license.controller'; -import '@/workflows/workflowHistory/workflowHistory.controller.ee'; +import '@/workflows/workflow-history/workflow-history.controller.ee'; import '@/workflows/workflows.controller'; +import { EventService } from './events/event.service'; const exec = promisify(callbackExec); @@ -81,16 +81,16 @@ export class Server extends AbstractServer { private readonly loadNodesAndCredentials: LoadNodesAndCredentials, private readonly orchestrationService: OrchestrationService, private readonly postHogClient: PostHogClient, - private readonly globalConfig: GlobalConfig, + private readonly eventService: EventService, ) { super('main'); this.testWebhooksEnabled = true; - this.webhooksEnabled = !config.getEnv('endpoints.disableProductionWebhooksOnMainProcess'); + this.webhooksEnabled = !this.globalConfig.endpoints.disableProductionWebhooksOnMainProcess; } async start() { - if (!config.getEnv('endpoints.disableUi')) { + if (!this.globalConfig.endpoints.disableUi) { const { FrontendService } = await import('@/services/frontend.service'); this.frontendService = Container.get(FrontendService); } @@ -106,7 +106,7 @@ export class Server extends AbstractServer { void this.loadNodesAndCredentials.setupHotReload(); } - void Container.get(InternalHooks).onServerStarted(); + this.eventService.emit('server-started'); } private async registerAdditionalControllers() { @@ -115,8 +115,8 @@ export class Server extends AbstractServer { } if (isLdapEnabled()) { - const { LdapService } = await import('@/Ldap/ldap.service.ee'); - await import('@/Ldap/ldap.controller.ee'); + const { LdapService } = await import('@/ldap/ldap.service.ee'); + await import('@/ldap/ldap.controller.ee'); await Container.get(LdapService).init(); } @@ -132,7 +132,7 @@ export class Server extends AbstractServer { await import('@/controllers/mfa.controller'); } - if (!config.getEnv('endpoints.disableUi')) { + if (!this.globalConfig.endpoints.disableUi) { await import('@/controllers/cta.controller'); } @@ -166,7 +166,7 @@ export class Server extends AbstractServer { } async configure(): Promise { - if (config.getEnv('endpoints.metrics.enable')) { + if (this.globalConfig.endpoints.metrics.enable) { const { PrometheusMetricsService } = await import('@/metrics/prometheus-metrics.service'); await Container.get(PrometheusMetricsService).init(this.app); } @@ -212,8 +212,8 @@ export class Server extends AbstractServer { setupPushHandler(restEndpoint, app); if (config.getEnv('executions.mode') === 'queue') { - const { Queue } = await import('@/Queue'); - await Container.get(Queue).init(); + const { ScalingService } = await import('@/scaling/scaling.service'); + await Container.get(ScalingService).setupQueue(); } await handleMfaDisable(); @@ -251,7 +251,7 @@ export class Server extends AbstractServer { // ---------------------------------------- const eventBus = Container.get(MessageEventBus); await eventBus.initialize(); - Container.get(AuditEventRelay).init(); + Container.get(LogStreamingEventRelay).init(); if (this.endpointPresetCredentials !== '') { // POST endpoint to set preset credentials @@ -306,7 +306,8 @@ export class Server extends AbstractServer { this.app.use('/icons/@:scope/:packageName/*/*.(svg|png)', serveIcons); this.app.use('/icons/:packageName/*/*.(svg|png)', serveIcons); - const isTLSEnabled = this.protocol === 'https' && !!(this.sslKey && this.sslCert); + const isTLSEnabled = + this.globalConfig.protocol === 'https' && !!(this.sslKey && this.sslCert); const isPreviewMode = process.env.N8N_PREVIEW_MODE === 'true'; const securityHeadersMiddleware = helmet({ contentSecurityPolicy: false, @@ -340,7 +341,7 @@ export class Server extends AbstractServer { this.restEndpoint, this.endpointPresetCredentials, isApiEnabled() ? '' : publicApiEndpoint, - ...config.getEnv('endpoints.additionalNonUIRoutes').split(':'), + ...this.globalConfig.endpoints.additionalNonUIRoutes.split(':'), ].filter((u) => !!u); const nonUIRoutesRegex = new RegExp(`^/(${nonUIRoutes.join('|')})/?.*$`); const historyApiHandler: express.RequestHandler = (req, res, next) => { diff --git a/packages/cli/test/unit/ExecutionMetadataService.test.ts b/packages/cli/src/services/__tests__/ExecutionMetadataService.test.ts similarity index 94% rename from packages/cli/test/unit/ExecutionMetadataService.test.ts rename to packages/cli/src/services/__tests__/ExecutionMetadataService.test.ts index 826aae5e25..8b77b8b168 100644 --- a/packages/cli/test/unit/ExecutionMetadataService.test.ts +++ b/packages/cli/src/services/__tests__/ExecutionMetadataService.test.ts @@ -1,7 +1,7 @@ import { Container } from 'typedi'; import { ExecutionMetadataRepository } from '@db/repositories/executionMetadata.repository'; import { ExecutionMetadataService } from '@/services/executionMetadata.service'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; describe('ExecutionMetadataService', () => { const repository = mockInstance(ExecutionMetadataRepository); diff --git a/packages/cli/test/unit/services/activeWorkflows.service.test.ts b/packages/cli/src/services/__tests__/activeWorkflows.service.test.ts similarity index 97% rename from packages/cli/test/unit/services/activeWorkflows.service.test.ts rename to packages/cli/src/services/__tests__/activeWorkflows.service.test.ts index 2089c94690..6b008e281b 100644 --- a/packages/cli/test/unit/services/activeWorkflows.service.test.ts +++ b/packages/cli/src/services/__tests__/activeWorkflows.service.test.ts @@ -1,4 +1,4 @@ -import type { ActivationErrorsService } from '@/ActivationErrors.service'; +import type { ActivationErrorsService } from '@/activation-errors.service'; import type { User } from '@db/entities/User'; import type { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import type { WorkflowRepository } from '@db/repositories/workflow.repository'; diff --git a/packages/cli/test/unit/services/communityPackages.service.test.ts b/packages/cli/src/services/__tests__/communityPackages.service.test.ts similarity index 86% rename from packages/cli/test/unit/services/communityPackages.service.test.ts rename to packages/cli/src/services/__tests__/communityPackages.service.test.ts index 2b0d8bf0d4..7b2431c172 100644 --- a/packages/cli/test/unit/services/communityPackages.service.test.ts +++ b/packages/cli/src/services/__tests__/communityPackages.service.test.ts @@ -2,8 +2,10 @@ import { exec } from 'child_process'; import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises'; import axios from 'axios'; import { mocked } from 'jest-mock'; -import Container from 'typedi'; +import { mock } from 'jest-mock-extended'; +import type { GlobalConfig } from '@n8n/config'; import type { PublicInstalledPackage } from 'n8n-workflow'; +import type { PackageDirectoryLoader } from 'n8n-core'; import { NODE_PACKAGE_PREFIX, @@ -11,24 +13,19 @@ import { NPM_PACKAGE_STATUS_GOOD, RESPONSE_ERROR_MESSAGES, } from '@/constants'; -import config from '@/config'; import { InstalledPackages } from '@db/entities/InstalledPackages'; import type { CommunityPackages } from '@/Interfaces'; import { CommunityPackagesService } from '@/services/communityPackages.service'; import { InstalledNodesRepository } from '@db/repositories/installedNodes.repository'; import { InstalledPackagesRepository } from '@db/repositories/installedPackages.repository'; import { InstalledNodes } from '@db/entities/InstalledNodes'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; +import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import type { License } from '@/license'; -import { mockInstance } from '../../shared/mocking'; -import { - COMMUNITY_NODE_VERSION, - COMMUNITY_PACKAGE_VERSION, -} from '../../integration/shared/constants'; -import { randomName } from '../../integration/shared/random'; -import { mockPackageName, mockPackagePair } from '../../integration/shared/utils'; -import { InstanceSettings, PackageDirectoryLoader } from 'n8n-core'; -import { Logger } from '@/Logger'; +import { mockInstance } from '@test/mocking'; +import { COMMUNITY_NODE_VERSION, COMMUNITY_PACKAGE_VERSION } from '@test-integration/constants'; +import { randomName } from '@test-integration/random'; +import { mockPackageName, mockPackagePair } from '@test-integration/utils'; jest.mock('fs/promises'); jest.mock('child_process'); @@ -43,10 +40,20 @@ const execMock = ((...args) => { }) as typeof exec; describe('CommunityPackagesService', () => { + const license = mock(); + const globalConfig = mock({ + nodes: { + communityPackages: { + reinstallMissing: false, + registry: 'some.random.host', + }, + }, + }); + const loadNodesAndCredentials = mock(); + + const nodeName = randomName(); const installedNodesRepository = mockInstance(InstalledNodesRepository); installedNodesRepository.create.mockImplementation(() => { - const nodeName = randomName(); - return Object.assign(new InstalledNodes(), { name: nodeName, type: nodeName, @@ -63,13 +70,15 @@ describe('CommunityPackagesService', () => { }); }); - mockInstance(LoadNodesAndCredentials); - - const communityPackagesService = Container.get(CommunityPackagesService); - - beforeEach(() => { - config.load(config.default); - }); + const communityPackagesService = new CommunityPackagesService( + mock(), + mock(), + mock(), + loadNodesAndCredentials, + mock(), + license, + globalConfig, + ); describe('parseNpmPackageName()', () => { test('should fail with empty package name', () => { @@ -368,50 +377,28 @@ describe('CommunityPackagesService', () => { }; describe('updateNpmModule', () => { - let packageDirectoryLoader: PackageDirectoryLoader; - let communityPackagesService: CommunityPackagesService; + const installedPackage = mock({ packageName: mockPackageName() }); + const packageDirectoryLoader = mock({ + loadedNodes: [{ name: nodeName, version: 1 }], + }); beforeEach(async () => { - jest.restoreAllMocks(); + jest.clearAllMocks(); - packageDirectoryLoader = mockInstance(PackageDirectoryLoader); - const loadNodesAndCredentials = mockInstance(LoadNodesAndCredentials); loadNodesAndCredentials.loadPackage.mockResolvedValue(packageDirectoryLoader); - const instanceSettings = mockInstance(InstanceSettings); - const logger = mockInstance(Logger); - const installedPackagesRepository = mockInstance(InstalledPackagesRepository); - - communityPackagesService = new CommunityPackagesService( - instanceSettings, - logger, - installedPackagesRepository, - loadNodesAndCredentials, - ); + mocked(exec).mockImplementation(execMock); }); - afterEach(async () => { - jest.restoreAllMocks(); - }); - - test('should call `exec` with the correct command ', async () => { + test('should call `exec` with the correct command and registry', async () => { // // ARRANGE // - const nodeName = randomName(); - packageDirectoryLoader.loadedNodes = [{ name: nodeName, version: 1 }]; - - const installedPackage = new InstalledPackages(); - installedPackage.packageName = mockPackageName(); - - mocked(exec).mockImplementation(execMock); + license.isCustomNpmRegistryEnabled.mockReturnValue(true); // // ACT // - await communityPackagesService.updateNpmModule( - installedPackage.packageName, - installedPackage, - ); + await communityPackagesService.updatePackage(installedPackage.packageName, installedPackage); // // ASSERT @@ -420,10 +407,32 @@ describe('CommunityPackagesService', () => { expect(exec).toHaveBeenCalledTimes(1); expect(exec).toHaveBeenNthCalledWith( 1, - `npm install ${installedPackage.packageName}@latest`, + `npm install ${installedPackage.packageName}@latest --registry=some.random.host`, expect.any(Object), expect.any(Function), ); }); + + test('should throw when not licensed', async () => { + // + // ARRANGE + // + license.isCustomNpmRegistryEnabled.mockReturnValue(false); + + // + // ACT + // + const promise = communityPackagesService.updatePackage( + installedPackage.packageName, + installedPackage, + ); + + // + // ASSERT + // + await expect(promise).rejects.toThrow( + 'Your license does not allow for feat:communityNodes:customRegistry.', + ); + }); }); }); diff --git a/packages/cli/test/unit/credentials-tester.unit.test.ts b/packages/cli/src/services/__tests__/credentials-tester.service.test.ts similarity index 90% rename from packages/cli/test/unit/credentials-tester.unit.test.ts rename to packages/cli/src/services/__tests__/credentials-tester.service.test.ts index e2987f5c84..e9f98c7a8b 100644 --- a/packages/cli/test/unit/credentials-tester.unit.test.ts +++ b/packages/cli/src/services/__tests__/credentials-tester.service.test.ts @@ -1,8 +1,8 @@ import { CredentialsTester } from '@/services/credentials-tester.service'; import mock from 'jest-mock-extended/lib/Mock'; -import type { CredentialTypes } from '@/CredentialTypes'; +import type { CredentialTypes } from '@/credential-types'; import type { ICredentialType, INodeType } from 'n8n-workflow'; -import type { NodeTypes } from '@/NodeTypes'; +import type { NodeTypes } from '@/node-types'; describe('CredentialsTester', () => { const credentialTypes = mock(); diff --git a/packages/cli/test/unit/services/curl.service.test.ts b/packages/cli/src/services/__tests__/curl.service.test.ts similarity index 100% rename from packages/cli/test/unit/services/curl.service.test.ts rename to packages/cli/src/services/__tests__/curl.service.test.ts diff --git a/packages/cli/test/unit/services/hooks.service.test.ts b/packages/cli/src/services/__tests__/hooks.service.test.ts similarity index 100% rename from packages/cli/test/unit/services/hooks.service.test.ts rename to packages/cli/src/services/__tests__/hooks.service.test.ts diff --git a/packages/cli/test/unit/services/jwt.service.test.ts b/packages/cli/src/services/__tests__/jwt.service.test.ts similarity index 100% rename from packages/cli/test/unit/services/jwt.service.test.ts rename to packages/cli/src/services/__tests__/jwt.service.test.ts diff --git a/packages/cli/test/unit/services/naming.service.test.ts b/packages/cli/src/services/__tests__/naming.service.test.ts similarity index 98% rename from packages/cli/test/unit/services/naming.service.test.ts rename to packages/cli/src/services/__tests__/naming.service.test.ts index ea2c34fb8c..1ca2167346 100644 --- a/packages/cli/test/unit/services/naming.service.test.ts +++ b/packages/cli/src/services/__tests__/naming.service.test.ts @@ -1,6 +1,6 @@ import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { NamingService } from '@/services/naming.service'; import type { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; import type { CredentialsEntity } from '@/databases/entities/CredentialsEntity'; diff --git a/packages/cli/test/unit/services/orchestration.service.test.ts b/packages/cli/src/services/__tests__/orchestration.service.test.ts similarity index 86% rename from packages/cli/test/unit/services/orchestration.service.test.ts rename to packages/cli/src/services/__tests__/orchestration.service.test.ts index 43aba8d484..99c92ad070 100644 --- a/packages/cli/test/unit/services/orchestration.service.test.ts +++ b/packages/cli/src/services/__tests__/orchestration.service.test.ts @@ -1,4 +1,9 @@ import Container from 'typedi'; +import type Redis from 'ioredis'; +import { mock } from 'jest-mock-extended'; +import { InstanceSettings } from 'n8n-core'; +import type { WorkflowActivateMode } from 'n8n-workflow'; + import config from '@/config'; import { OrchestrationService } from '@/services/orchestration.service'; import type { RedisServiceWorkerResponseObject } from '@/services/redis/RedisServiceCommands'; @@ -8,16 +13,15 @@ import { handleWorkerResponseMessageMain } from '@/services/orchestration/main/h import { handleCommandMessageMain } from '@/services/orchestration/main/handleCommandMessageMain'; import { OrchestrationHandlerMainService } from '@/services/orchestration/main/orchestration.handler.main.service'; import * as helpers from '@/services/orchestration/helpers'; -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { Logger } from '@/Logger'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; +import { Logger } from '@/logger'; import { Push } from '@/push'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; -import { mockInstance } from '../../shared/mocking'; -import type { WorkflowActivateMode } from 'n8n-workflow'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; +import { mockInstance } from '@test/mocking'; import { RedisClientService } from '@/services/redis/redis-client.service'; -import type Redis from 'ioredis'; -import { mock } from 'jest-mock-extended'; +import type { MainResponseReceivedHandlerOptions } from '../orchestration/main/types'; +const instanceSettings = Container.get(InstanceSettings); const redisClientService = mockInstance(RedisClientService); const mockRedisClient = mock(); redisClientService.createClient.mockReturnValue(mockRedisClient); @@ -33,7 +37,7 @@ function setDefaultConfig() { config.set('generic.instanceType', 'main'); } -const workerRestartEventbusResponse: RedisServiceWorkerResponseObject = { +const workerRestartEventBusResponse: RedisServiceWorkerResponseObject = { senderId: 'test', workerId: 'test', command: 'restartEventBus', @@ -72,6 +76,10 @@ describe('Orchestration Service', () => { queueModeId = config.get('redis.queueModeId'); }); + beforeEach(() => { + instanceSettings.markAsLeader(); + }); + afterAll(async () => { jest.mock('@/services/redis/RedisServicePubSubPublisher').restoreAllMocks(); jest.mock('@/services/redis/RedisServicePubSubSubscriber').restoreAllMocks(); @@ -88,9 +96,10 @@ describe('Orchestration Service', () => { test('should handle worker responses', async () => { const response = await handleWorkerResponseMessageMain( - JSON.stringify(workerRestartEventbusResponse), + JSON.stringify(workerRestartEventBusResponse), + mock(), ); - expect(response.command).toEqual('restartEventBus'); + expect(response?.command).toEqual('restartEventBus'); }); test('should handle command messages from others', async () => { @@ -108,7 +117,7 @@ describe('Orchestration Service', () => { test('should reject command messages from itself', async () => { const response = await handleCommandMessageMain( - JSON.stringify({ ...workerRestartEventbusResponse, senderId: queueModeId }), + JSON.stringify({ ...workerRestartEventBusResponse, senderId: queueModeId }), ); expect(response).toBeDefined(); expect(response!.command).toEqual('restartEventBus'); @@ -141,13 +150,10 @@ describe('Orchestration Service', () => { ); expect(helpers.debounceMessageReceiver).toHaveBeenCalledTimes(2); expect(res1!.payload).toBeUndefined(); - expect(res2!.payload!.result).toEqual('debounced'); + expect(res2!.payload).toEqual({ result: 'debounced' }); }); describe('shouldAddWebhooks', () => { - beforeEach(() => { - config.set('multiMainSetup.instanceType', 'leader'); - }); test('should return true for init', () => { // We want to ensure that webhooks are populated on init // more https://github.com/n8n-io/n8n/pull/8830 @@ -169,7 +175,7 @@ describe('Orchestration Service', () => { }); test('should return false for update or activate when not leader', () => { - config.set('multiMainSetup.instanceType', 'follower'); + instanceSettings.markAsFollower(); const modes = ['update', 'activate'] as WorkflowActivateMode[]; for (const mode of modes) { const result = os.shouldAddWebhooks(mode); diff --git a/packages/cli/test/unit/services/ownership.service.test.ts b/packages/cli/src/services/__tests__/ownership.service.test.ts similarity index 98% rename from packages/cli/test/unit/services/ownership.service.test.ts rename to packages/cli/src/services/__tests__/ownership.service.test.ts index d1a722da19..8a3d40eb60 100644 --- a/packages/cli/test/unit/services/ownership.service.test.ts +++ b/packages/cli/src/services/__tests__/ownership.service.test.ts @@ -3,14 +3,14 @@ import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.reposi import { SharedWorkflow } from '@db/entities/SharedWorkflow'; import { User } from '@db/entities/User'; import type { SharedCredentials } from '@db/entities/SharedCredentials'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; import { UserRepository } from '@/databases/repositories/user.repository'; import { mock } from 'jest-mock-extended'; import { Project } from '@/databases/entities/Project'; import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; import { ProjectRelation } from '@/databases/entities/ProjectRelation'; -import { mockCredential, mockProject } from '../shared/mockObjects'; +import { mockCredential, mockProject } from '@test/mockObjects'; describe('OwnershipService', () => { const userRepository = mockInstance(UserRepository); diff --git a/packages/cli/test/unit/utilities/password.utility.test.ts b/packages/cli/src/services/__tests__/password.utility.test.ts similarity index 100% rename from packages/cli/test/unit/utilities/password.utility.test.ts rename to packages/cli/src/services/__tests__/password.utility.test.ts diff --git a/packages/cli/test/unit/services/redis.service.test.ts b/packages/cli/src/services/__tests__/redis.service.test.ts similarity index 92% rename from packages/cli/test/unit/services/redis.service.test.ts rename to packages/cli/src/services/__tests__/redis.service.test.ts index cb963ad535..1e282e6df1 100644 --- a/packages/cli/test/unit/services/redis.service.test.ts +++ b/packages/cli/src/services/__tests__/redis.service.test.ts @@ -1,8 +1,8 @@ import Container from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import config from '@/config'; import { RedisService } from '@/services/redis.service'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; jest.mock('ioredis', () => { const Redis = require('ioredis-mock'); @@ -14,7 +14,7 @@ jest.mock('ioredis', () => { }; } // second mock for our code - return function (...args: any) { + return function (...args: unknown[]) { return new Redis(args); }; }); diff --git a/packages/cli/test/unit/services/user.service.test.ts b/packages/cli/src/services/__tests__/user.service.test.ts similarity index 90% rename from packages/cli/test/unit/services/user.service.test.ts rename to packages/cli/src/services/__tests__/user.service.test.ts index 9f84b1d202..5aeb919220 100644 --- a/packages/cli/test/unit/services/user.service.test.ts +++ b/packages/cli/src/services/__tests__/user.service.test.ts @@ -4,13 +4,21 @@ import { v4 as uuid } from 'uuid'; import { User } from '@db/entities/User'; import { UserService } from '@/services/user.service'; import { UrlService } from '@/services/url.service'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { UserRepository } from '@/databases/repositories/user.repository'; +import { GlobalConfig } from '@n8n/config'; describe('UserService', () => { - const urlService = new UrlService(); + const globalConfig = mockInstance(GlobalConfig, { + host: 'localhost', + path: '/', + port: 5678, + listen_address: '0.0.0.0', + protocol: 'http', + }); + const urlService = new UrlService(globalConfig); const userRepository = mockInstance(UserRepository); - const userService = new UserService(mock(), userRepository, mock(), urlService); + const userService = new UserService(mock(), userRepository, mock(), urlService, mock()); const commonMockUser = Object.assign(new User(), { id: uuid(), diff --git a/packages/cli/test/unit/services/workflow-statistics.service.test.ts b/packages/cli/src/services/__tests__/workflow-statistics.service.test.ts similarity index 80% rename from packages/cli/test/unit/services/workflow-statistics.service.test.ts rename to packages/cli/src/services/__tests__/workflow-statistics.service.test.ts index da1338f8eb..b3cc6b7998 100644 --- a/packages/cli/test/unit/services/workflow-statistics.service.test.ts +++ b/packages/cli/src/services/__tests__/workflow-statistics.service.test.ts @@ -17,8 +17,9 @@ import { WorkflowStatisticsRepository } from '@db/repositories/workflowStatistic import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; import { UserService } from '@/services/user.service'; import { OwnershipService } from '@/services/ownership.service'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import type { Project } from '@/databases/entities/Project'; +import type { EventService } from '@/events/event.service'; describe('WorkflowStatisticsService', () => { const fakeUser = mock({ id: 'abcde-fghij' }); @@ -44,20 +45,15 @@ describe('WorkflowStatisticsService', () => { mocked(ownershipService.getProjectOwnerCached).mockResolvedValue(fakeUser); const updateSettingsMock = jest.spyOn(userService, 'updateSettings').mockImplementation(); + const eventService = mock(); const workflowStatisticsService = new WorkflowStatisticsService( mock(), new WorkflowStatisticsRepository(dataSource, globalConfig), ownershipService, + userService, + eventService, ); - const onFirstProductionWorkflowSuccess = jest.fn(); - const onFirstWorkflowDataLoad = jest.fn(); - workflowStatisticsService.on( - 'telemetry.onFirstProductionWorkflowSuccess', - onFirstProductionWorkflowSuccess, - ); - workflowStatisticsService.on('telemetry.onFirstWorkflowDataLoad', onFirstWorkflowDataLoad); - beforeEach(() => { jest.clearAllMocks(); }); @@ -96,11 +92,10 @@ describe('WorkflowStatisticsService', () => { await workflowStatisticsService.workflowExecutionCompleted(workflow, runData); expect(updateSettingsMock).toHaveBeenCalledTimes(1); - expect(onFirstProductionWorkflowSuccess).toBeCalledTimes(1); - expect(onFirstProductionWorkflowSuccess).toHaveBeenNthCalledWith(1, { - project_id: fakeProject.id, - user_id: fakeUser.id, - workflow_id: workflow.id, + expect(eventService.emit).toHaveBeenCalledWith('first-production-workflow-succeeded', { + projectId: fakeProject.id, + workflowId: workflow.id, + userId: fakeUser.id, }); }); @@ -117,13 +112,13 @@ describe('WorkflowStatisticsService', () => { }; const runData: IRun = { finished: false, - status: 'failed', + status: 'error', data: { resultData: { runData: {} } }, mode: 'internal' as WorkflowExecuteMode, startedAt: new Date(), }; await workflowStatisticsService.workflowExecutionCompleted(workflow, runData); - expect(onFirstProductionWorkflowSuccess).toBeCalledTimes(0); + expect(eventService.emit).not.toHaveBeenCalled(); }); test('should not send metrics for updated entries', async () => { @@ -146,7 +141,7 @@ describe('WorkflowStatisticsService', () => { }; mockDBCall(2); await workflowStatisticsService.workflowExecutionCompleted(workflow, runData); - expect(onFirstProductionWorkflowSuccess).toBeCalledTimes(0); + expect(eventService.emit).not.toHaveBeenCalled(); }); }); @@ -163,13 +158,12 @@ describe('WorkflowStatisticsService', () => { parameters: {}, }; await workflowStatisticsService.nodeFetchedData(workflowId, node); - expect(onFirstWorkflowDataLoad).toBeCalledTimes(1); - expect(onFirstWorkflowDataLoad).toHaveBeenNthCalledWith(1, { - user_id: fakeUser.id, - project_id: fakeProject.id, - workflow_id: workflowId, - node_type: node.type, - node_id: node.id, + expect(eventService.emit).toHaveBeenCalledWith('first-workflow-data-loaded', { + userId: fakeUser.id, + project: fakeProject.id, + workflowId, + nodeType: node.type, + nodeId: node.id, }); }); @@ -191,21 +185,20 @@ describe('WorkflowStatisticsService', () => { }, }; await workflowStatisticsService.nodeFetchedData(workflowId, node); - expect(onFirstWorkflowDataLoad).toBeCalledTimes(1); - expect(onFirstWorkflowDataLoad).toHaveBeenNthCalledWith(1, { - user_id: fakeUser.id, - project_id: fakeProject.id, - workflow_id: workflowId, - node_type: node.type, - node_id: node.id, - credential_type: 'testCredentials', - credential_id: node.credentials.testCredentials.id, + expect(eventService.emit).toHaveBeenCalledWith('first-workflow-data-loaded', { + userId: fakeUser.id, + project: fakeProject.id, + workflowId, + nodeType: node.type, + nodeId: node.id, + credentialType: 'testCredentials', + credentialId: node.credentials.testCredentials.id, }); }); test('should not send metrics for entries that already have the flag set', async () => { // Fetch data for workflow 2 which is set up to not be altered in the mocks - entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, '')); + entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, new Error())); const workflowId = '1'; const node = { id: 'abcde', @@ -216,7 +209,7 @@ describe('WorkflowStatisticsService', () => { parameters: {}, }; await workflowStatisticsService.nodeFetchedData(workflowId, node); - expect(onFirstWorkflowDataLoad).toBeCalledTimes(0); + expect(eventService.emit).not.toHaveBeenCalled(); }); }); }); diff --git a/packages/cli/src/services/activeWorkflows.service.ts b/packages/cli/src/services/activeWorkflows.service.ts index ae2c083d72..c900795a3a 100644 --- a/packages/cli/src/services/activeWorkflows.service.ts +++ b/packages/cli/src/services/activeWorkflows.service.ts @@ -3,9 +3,9 @@ import { Service } from 'typedi'; import type { User } from '@db/entities/User'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { ActivationErrorsService } from '@/ActivationErrors.service'; +import { ActivationErrorsService } from '@/activation-errors.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; @Service() export class ActiveWorkflowsService { diff --git a/packages/cli/src/services/aiAsisstant.service.ts b/packages/cli/src/services/aiAsisstant.service.ts new file mode 100644 index 0000000000..62c4778cfb --- /dev/null +++ b/packages/cli/src/services/aiAsisstant.service.ts @@ -0,0 +1,54 @@ +import { Service } from 'typedi'; +import type { AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; +import { AiAssistantClient } from '@n8n_io/ai-assistant-sdk'; +import { assert, type IUser } from 'n8n-workflow'; +import { License } from '../license'; +import { N8N_VERSION } from '../constants'; +import config from '@/config'; +import type { AiAssistantRequest } from '@/requests'; +import type { Response } from 'undici'; + +@Service() +export class AiAssistantService { + private client: AiAssistantClient | undefined; + + constructor(private readonly licenseService: License) {} + + async init() { + const aiAssistantEnabled = this.licenseService.isAiAssistantEnabled(); + if (!aiAssistantEnabled) { + return; + } + + const licenseCert = await this.licenseService.loadCertStr(); + const consumerId = this.licenseService.getConsumerId(); + const baseUrl = config.get('aiAssistant.baseUrl'); + const logLevel = config.getEnv('logs.level'); + + this.client = new AiAssistantClient({ + licenseCert, + consumerId, + n8nVersion: N8N_VERSION, + baseUrl, + logLevel, + }); + } + + async chat(payload: AiAssistantSDK.ChatRequestPayload, user: IUser): Promise { + if (!this.client) { + await this.init(); + } + assert(this.client, 'Assistant client not setup'); + + return await this.client.chat(payload, { id: user.id }); + } + + async applySuggestion(payload: AiAssistantRequest.SuggestionPayload, user: IUser) { + if (!this.client) { + await this.init(); + } + assert(this.client, 'Assistant client not setup'); + + return await this.client.applySuggestion(payload, { id: user.id }); + } +} diff --git a/packages/cli/test/unit/services/cache-mock.service.test.ts b/packages/cli/src/services/cache/__tests__/cache-mock.service.test.ts similarity index 100% rename from packages/cli/test/unit/services/cache-mock.service.test.ts rename to packages/cli/src/services/cache/__tests__/cache-mock.service.test.ts diff --git a/packages/cli/test/unit/services/cache.service.test.ts b/packages/cli/src/services/cache/__tests__/cache.service.test.ts similarity index 92% rename from packages/cli/test/unit/services/cache.service.test.ts rename to packages/cli/src/services/cache/__tests__/cache.service.test.ts index a742b20698..5c024b2903 100644 --- a/packages/cli/test/unit/services/cache.service.test.ts +++ b/packages/cli/src/services/cache/__tests__/cache.service.test.ts @@ -1,6 +1,8 @@ import { CacheService } from '@/services/cache/cache.service'; import config from '@/config'; import { sleep } from 'n8n-workflow'; +import { GlobalConfig } from '@n8n/config'; +import Container from 'typedi'; jest.mock('ioredis', () => { const Redis = require('ioredis-mock'); @@ -13,10 +15,12 @@ jest.mock('ioredis', () => { for (const backend of ['memory', 'redis'] as const) { describe(backend, () => { let cacheService: CacheService; + let globalConfig: GlobalConfig; beforeAll(async () => { - config.set('cache.backend', backend); - cacheService = new CacheService(); + globalConfig = Container.get(GlobalConfig); + globalConfig.cache.backend = backend; + cacheService = new CacheService(globalConfig); await cacheService.init(); }); @@ -43,7 +47,7 @@ for (const backend of ['memory', 'redis'] as const) { if (backend === 'memory') { test('should honor max size when enough', async () => { - config.set('cache.memory.maxSize', 16); // enough bytes for "withoutUnicode" + globalConfig.cache.memory.maxSize = 16; // enough bytes for "withoutUnicode" await cacheService.init(); await cacheService.set('key', 'withoutUnicode'); @@ -51,12 +55,12 @@ for (const backend of ['memory', 'redis'] as const) { await expect(cacheService.get('key')).resolves.toBe('withoutUnicode'); // restore - config.set('cache.memory.maxSize', 3 * 1024 * 1024); + globalConfig.cache.memory.maxSize = 3 * 1024 * 1024; await cacheService.init(); }); test('should honor max size when not enough', async () => { - config.set('cache.memory.maxSize', 16); // not enough bytes for "withUnicodeԱԲԳ" + globalConfig.cache.memory.maxSize = 16; // not enough bytes for "withUnicodeԱԲԳ" await cacheService.init(); await cacheService.set('key', 'withUnicodeԱԲԳ'); @@ -64,7 +68,8 @@ for (const backend of ['memory', 'redis'] as const) { await expect(cacheService.get('key')).resolves.toBeUndefined(); // restore - config.set('cache.memory.maxSize', 3 * 1024 * 1024); + globalConfig.cache.memory.maxSize = 3 * 1024 * 1024; + // restore await cacheService.init(); }); } diff --git a/packages/cli/src/services/cache/cache.service.ts b/packages/cli/src/services/cache/cache.service.ts index 8e9a4dc95c..eaa58c32f0 100644 --- a/packages/cli/src/services/cache/cache.service.ts +++ b/packages/cli/src/services/cache/cache.service.ts @@ -1,5 +1,3 @@ -import EventEmitter from 'node:events'; - import Container, { Service } from 'typedi'; import { caching } from 'cache-manager'; import { ApplicationError, jsonStringify } from 'n8n-workflow'; @@ -10,20 +8,30 @@ import { MalformedRefreshValueError } from '@/errors/cache-errors/malformed-refr import type { TaggedRedisCache, TaggedMemoryCache, - CacheEvent, MaybeHash, Hash, } from '@/services/cache/cache.types'; import { TIME } from '@/constants'; +import { TypedEmitter } from '@/TypedEmitter'; +import { GlobalConfig } from '@n8n/config'; + +type CacheEvents = { + 'metrics.cache.hit': never; + 'metrics.cache.miss': never; + 'metrics.cache.update': never; +}; @Service() -export class CacheService extends EventEmitter { +export class CacheService extends TypedEmitter { + constructor(private readonly globalConfig: GlobalConfig) { + super(); + } + private cache: TaggedRedisCache | TaggedMemoryCache; async init() { - const backend = config.getEnv('cache.backend'); + const { backend } = this.globalConfig.cache; const mode = config.getEnv('executions.mode'); - const ttl = config.getEnv('cache.redis.ttl'); const useRedis = backend === 'redis' || (backend === 'auto' && mode === 'queue'); @@ -32,16 +40,19 @@ export class CacheService extends EventEmitter { const redisClientService = Container.get(RedisClientService); const prefixBase = config.getEnv('redis.prefix'); - const cachePrefix = config.getEnv('cache.redis.prefix'); - const prefix = redisClientService.toValidPrefix(`${prefixBase}:${cachePrefix}:`); + const prefix = redisClientService.toValidPrefix( + `${prefixBase}:${this.globalConfig.cache.redis.prefix}:`, + ); const redisClient = redisClientService.createClient({ - type: 'client(cache)', + type: 'cache(n8n)', extraOptions: { keyPrefix: prefix }, }); const { redisStoreUsingClient } = await import('@/services/cache/redis.cache-manager'); - const redisStore = redisStoreUsingClient(redisClient, { ttl }); + const redisStore = redisStoreUsingClient(redisClient, { + ttl: this.globalConfig.cache.redis.ttl, + }); const redisCache = await caching(redisStore); @@ -50,7 +61,7 @@ export class CacheService extends EventEmitter { return; } - const maxSize = config.getEnv('cache.memory.maxSize'); + const { maxSize, ttl } = this.globalConfig.cache.memory; const sizeCalculation = (item: unknown) => { const str = jsonStringify(item, { replaceCircularRefs: true }); @@ -66,10 +77,6 @@ export class CacheService extends EventEmitter { await this.cache.store.reset(); } - emit(event: CacheEvent, ...args: unknown[]) { - return super.emit(event, ...args); - } - isRedis() { return this.cache.kind === 'redis'; } diff --git a/packages/cli/src/services/cache/cache.types.ts b/packages/cli/src/services/cache/cache.types.ts index 4e96b8012a..f598e22494 100644 --- a/packages/cli/src/services/cache/cache.types.ts +++ b/packages/cli/src/services/cache/cache.types.ts @@ -8,5 +8,3 @@ export type TaggedMemoryCache = MemoryCache & { kind: 'memory' }; export type Hash = Record; export type MaybeHash = Hash | undefined; - -export type CacheEvent = `metrics.cache.${'hit' | 'miss' | 'update'}`; diff --git a/packages/cli/src/services/communityPackages.service.ts b/packages/cli/src/services/communityPackages.service.ts index 23bf9461d6..bfe7f6fef8 100644 --- a/packages/cli/src/services/communityPackages.service.ts +++ b/packages/cli/src/services/communityPackages.service.ts @@ -5,6 +5,7 @@ import { Service } from 'typedi'; import { promisify } from 'util'; import axios from 'axios'; +import { GlobalConfig } from '@n8n/config'; import { ApplicationError, type PublicInstalledPackage } from 'n8n-workflow'; import { InstanceSettings } from 'n8n-core'; import type { PackageDirectoryLoader } from 'n8n-core'; @@ -13,15 +14,21 @@ import { toError } from '@/utils'; import { InstalledPackagesRepository } from '@db/repositories/installedPackages.repository'; import type { InstalledPackages } from '@db/entities/InstalledPackages'; import { + LICENSE_FEATURES, NODE_PACKAGE_PREFIX, NPM_COMMAND_TOKENS, NPM_PACKAGE_STATUS_GOOD, RESPONSE_ERROR_MESSAGES, UNKNOWN_FAILURE_REASON, } from '@/constants'; +import { FeatureNotLicensedError } from '@/errors/feature-not-licensed.error'; import type { CommunityPackages } from '@/Interfaces'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; -import { Logger } from '@/Logger'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { Logger } from '@/logger'; +import { OrchestrationService } from './orchestration.service'; +import { License } from '@/license'; + +const DEFAULT_REGISTRY = 'https://registry.npmjs.org'; const { PACKAGE_NAME_NOT_PROVIDED, @@ -45,6 +52,8 @@ const INVALID_OR_SUSPICIOUS_PACKAGE_NAME = /[^0-9a-z@\-./]/; @Service() export class CommunityPackagesService { + reinstallMissingPackages = false; + missingPackages: string[] = []; constructor( @@ -52,6 +61,9 @@ export class CommunityPackagesService { private readonly logger: Logger, private readonly installedPackageRepository: InstalledPackagesRepository, private readonly loadNodesAndCredentials: LoadNodesAndCredentials, + private readonly orchestrationService: OrchestrationService, + private readonly license: License, + private readonly globalConfig: GlobalConfig, ) {} get hasMissingPackages() { @@ -73,11 +85,11 @@ export class CommunityPackagesService { return await this.installedPackageRepository.find({ relations: ['installedNodes'] }); } - async removePackageFromDatabase(packageName: InstalledPackages) { + private async removePackageFromDatabase(packageName: InstalledPackages) { return await this.installedPackageRepository.remove(packageName); } - async persistInstalledPackage(packageLoader: PackageDirectoryLoader) { + private async persistInstalledPackage(packageLoader: PackageDirectoryLoader) { try { return await this.installedPackageRepository.saveInstalledPackageWithNodes(packageLoader); } catch (maybeError) { @@ -251,7 +263,7 @@ export class CommunityPackagesService { } } - async setMissingPackages({ reinstallMissingPackages }: { reinstallMissingPackages: boolean }) { + async checkForMissingPackages() { const installedPackages = await this.getAllInstalledPackages(); const missingPackages = new Set<{ packageName: string; version: string }>(); @@ -271,24 +283,25 @@ export class CommunityPackagesService { if (missingPackages.size === 0) return; - this.logger.error( - 'n8n detected that some packages are missing. For more information, visit https://docs.n8n.io/integrations/community-nodes/troubleshooting/', - ); - - if (reinstallMissingPackages || process.env.N8N_REINSTALL_MISSING_PACKAGES) { + const { reinstallMissing } = this.globalConfig.nodes.communityPackages; + if (reinstallMissing) { this.logger.info('Attempting to reinstall missing packages', { missingPackages }); try { // Optimistic approach - stop if any installation fails - for (const missingPackage of missingPackages) { - await this.installNpmModule(missingPackage.packageName, missingPackage.version); + await this.installPackage(missingPackage.packageName, missingPackage.version); missingPackages.delete(missingPackage); } this.logger.info('Packages reinstalled successfully. Resuming regular initialization.'); + await this.loadNodesAndCredentials.postProcessLoaders(); } catch (error) { this.logger.error('n8n was unable to install the missing packages.'); } + } else { + this.logger.warn( + 'n8n detected that some packages are missing. For more information, visit https://docs.n8n.io/integrations/community-nodes/troubleshooting/', + ); } this.missingPackages = [...missingPackages].map( @@ -296,32 +309,38 @@ export class CommunityPackagesService { ); } - async installNpmModule(packageName: string, version?: string): Promise { - return await this.installOrUpdateNpmModule(packageName, { version }); + async installPackage(packageName: string, version?: string): Promise { + return await this.installOrUpdatePackage(packageName, { version }); } - async updateNpmModule( + async updatePackage( packageName: string, installedPackage: InstalledPackages, ): Promise { - return await this.installOrUpdateNpmModule(packageName, { installedPackage }); + return await this.installOrUpdatePackage(packageName, { installedPackage }); } - async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise { - await this.executeNpmCommand(`npm remove ${packageName}`); + async removePackage(packageName: string, installedPackage: InstalledPackages): Promise { + await this.removeNpmPackage(packageName); await this.removePackageFromDatabase(installedPackage); - await this.loadNodesAndCredentials.unloadPackage(packageName); - await this.loadNodesAndCredentials.postProcessLoaders(); + await this.orchestrationService.publish('community-package-uninstall', { packageName }); } - private async installOrUpdateNpmModule( + private getNpmRegistry() { + const { registry } = this.globalConfig.nodes.communityPackages; + if (registry !== DEFAULT_REGISTRY && !this.license.isCustomNpmRegistryEnabled()) { + throw new FeatureNotLicensedError(LICENSE_FEATURES.COMMUNITY_NODES_CUSTOM_REGISTRY); + } + return registry; + } + + private async installOrUpdatePackage( packageName: string, options: { version?: string } | { installedPackage: InstalledPackages }, ) { const isUpdate = 'installedPackage' in options; - const command = isUpdate - ? `npm install ${packageName}@latest` - : `npm install ${packageName}${options.version ? `@${options.version}` : ''}`; + const packageVersion = isUpdate || !options.version ? 'latest' : options.version; + const command = `npm install ${packageName}@${packageVersion} --registry=${this.getNpmRegistry()}`; try { await this.executeNpmCommand(command); @@ -337,9 +356,8 @@ export class CommunityPackagesService { loader = await this.loadNodesAndCredentials.loadPackage(packageName); } catch (error) { // Remove this package since loading it failed - const removeCommand = `npm remove ${packageName}`; try { - await this.executeNpmCommand(removeCommand); + await this.executeNpmCommand(`npm remove ${packageName}`); } catch {} throw new ApplicationError(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error }); } @@ -351,7 +369,12 @@ export class CommunityPackagesService { await this.removePackageFromDatabase(options.installedPackage); } const installedPackage = await this.persistInstalledPackage(loader); + await this.orchestrationService.publish( + isUpdate ? 'community-package-update' : 'community-package-install', + { packageName, packageVersion }, + ); await this.loadNodesAndCredentials.postProcessLoaders(); + this.logger.info(`Community package installed: ${packageName}`); return installedPackage; } catch (error) { throw new ApplicationError('Failed to save installed package', { @@ -361,12 +384,26 @@ export class CommunityPackagesService { } } else { // Remove this package since it contains no loadable nodes - const removeCommand = `npm remove ${packageName}`; try { - await this.executeNpmCommand(removeCommand); + await this.executeNpmCommand(`npm remove ${packageName}`); } catch {} - throw new ApplicationError(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES); } } + + async installOrUpdateNpmPackage(packageName: string, packageVersion: string) { + await this.executeNpmCommand( + `npm install ${packageName}@${packageVersion} --registry=${this.getNpmRegistry()}`, + ); + await this.loadNodesAndCredentials.loadPackage(packageName); + await this.loadNodesAndCredentials.postProcessLoaders(); + this.logger.info(`Community package installed: ${packageName}`); + } + + async removeNpmPackage(packageName: string) { + await this.executeNpmCommand(`npm remove ${packageName}`); + await this.loadNodesAndCredentials.unloadPackage(packageName); + await this.loadNodesAndCredentials.postProcessLoaders(); + this.logger.info(`Community package uninstalled: ${packageName}`); + } } diff --git a/packages/cli/src/services/credentials-tester.service.ts b/packages/cli/src/services/credentials-tester.service.ts index c9ddc7b15e..4f3475d303 100644 --- a/packages/cli/src/services/credentials-tester.service.ts +++ b/packages/cli/src/services/credentials-tester.service.ts @@ -34,14 +34,14 @@ import { ApplicationError, } from 'n8n-workflow'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import type { User } from '@db/entities/User'; -import { NodeTypes } from '@/NodeTypes'; -import { CredentialTypes } from '@/CredentialTypes'; +import { NodeTypes } from '@/node-types'; +import { CredentialTypes } from '@/credential-types'; import { RESPONSE_ERROR_MESSAGES } from '../constants'; import { isObjectLiteral } from '../utils'; -import { Logger } from '@/Logger'; -import { CredentialsHelper } from '../CredentialsHelper'; +import { Logger } from '@/logger'; +import { CredentialsHelper } from '../credentials-helper'; const { OAUTH2_CREDENTIAL_TEST_SUCCEEDED, OAUTH2_CREDENTIAL_TEST_FAILED } = RESPONSE_ERROR_MESSAGES; diff --git a/packages/cli/src/services/dynamicNodeParameters.service.ts b/packages/cli/src/services/dynamicNodeParameters.service.ts index 1788bac6e1..d69f789373 100644 --- a/packages/cli/src/services/dynamicNodeParameters.service.ts +++ b/packages/cli/src/services/dynamicNodeParameters.service.ts @@ -15,10 +15,12 @@ import type { INodeCredentials, INodeParameters, INodeTypeNameVersion, + NodeParameterValueType, + IDataObject, } from 'n8n-workflow'; import { Workflow, RoutingNode, ApplicationError } from 'n8n-workflow'; import { NodeExecuteFunctions } from 'n8n-core'; -import { NodeTypes } from '@/NodeTypes'; +import { NodeTypes } from '@/node-types'; @Service() export class DynamicNodeParametersService { @@ -156,6 +158,24 @@ export class DynamicNodeParametersService { return method.call(thisArgs); } + /** Returns the result of the action handler */ + async getActionResult( + handler: string, + path: string, + additionalData: IWorkflowExecuteAdditionalData, + nodeTypeAndVersion: INodeTypeNameVersion, + currentNodeParameters: INodeParameters, + payload: IDataObject | string | undefined, + credentials?: INodeCredentials, + ): Promise { + const nodeType = this.getNodeType(nodeTypeAndVersion); + const method = this.getMethod('actionHandler', handler, nodeType); + const workflow = this.getWorkflow(nodeTypeAndVersion, currentNodeParameters, credentials); + const thisArgs = this.getThisArg(path, additionalData, workflow); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return method.call(thisArgs, payload); + } + private getMethod( type: 'resourceMapping', methodName: string, @@ -175,9 +195,14 @@ export class DynamicNodeParametersService { methodName: string, nodeType: INodeType, ): (this: ILoadOptionsFunctions) => Promise; + private getMethod( + type: 'actionHandler', + methodName: string, + nodeType: INodeType, + ): (this: ILoadOptionsFunctions, payload?: string) => Promise; private getMethod( - type: 'resourceMapping' | 'listSearch' | 'loadOptions', + type: 'resourceMapping' | 'listSearch' | 'loadOptions' | 'actionHandler', methodName: string, nodeType: INodeType, ) { diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index 918de17935..24f8a3505b 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -15,23 +15,23 @@ import { InstanceSettings } from 'n8n-core'; import config from '@/config'; import { LICENSE_FEATURES } from '@/constants'; -import { CredentialsOverwrites } from '@/CredentialsOverwrites'; -import { CredentialTypes } from '@/CredentialTypes'; -import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; -import { License } from '@/License'; -import { getCurrentAuthenticationMethod } from '@/sso/ssoHelpers'; -import { getLdapLoginLabel } from '@/Ldap/helpers.ee'; -import { getSamlLoginLabel } from '@/sso/saml/samlHelpers'; +import { CredentialsOverwrites } from '@/credentials-overwrites'; +import { CredentialTypes } from '@/credential-types'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { License } from '@/license'; +import { getCurrentAuthenticationMethod } from '@/sso/sso-helpers'; +import { getLdapLoginLabel } from '@/ldap/helpers.ee'; +import { getSamlLoginLabel } from '@/sso/saml/saml-helpers'; import { getVariablesLimit } from '@/environments/variables/environmentHelpers'; import { getWorkflowHistoryLicensePruneTime, getWorkflowHistoryPruneTime, -} from '@/workflows/workflowHistory/workflowHistoryHelper.ee'; -import { UserManagementMailer } from '@/UserManagement/email'; +} from '@/workflows/workflow-history/workflow-history-helper.ee'; +import { UserManagementMailer } from '@/user-management/email'; import type { CommunityPackagesService } from '@/services/communityPackages.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { UrlService } from './url.service'; -import { InternalHooks } from '@/InternalHooks'; +import { EventService } from '@/events/event.service'; import { isApiEnabled } from '@/PublicApi'; @Service() @@ -50,7 +50,7 @@ export class FrontendService { private readonly mailer: UserManagementMailer, private readonly instanceSettings: InstanceSettings, private readonly urlService: UrlService, - private readonly internalHooks: InternalHooks, + private readonly eventService: EventService, ) { loadNodesAndCredentials.addPostProcessor(async () => await this.generateTypes()); void this.generateTypes(); @@ -66,7 +66,7 @@ export class FrontendService { private initSettings() { const instanceBaseUrl = this.urlService.getInstanceBaseUrl(); - const restEndpoint = config.getEnv('endpoints.rest'); + const restEndpoint = this.globalConfig.endpoints.rest; const telemetrySettings: ITelemetrySettings = { enabled: config.getEnv('diagnostics.enabled'), @@ -88,11 +88,11 @@ export class FrontendService { isDocker: this.isDocker(), databaseType: this.globalConfig.database.type, previewMode: process.env.N8N_PREVIEW_MODE === 'true', - endpointForm: config.getEnv('endpoints.form'), - endpointFormTest: config.getEnv('endpoints.formTest'), - endpointFormWaiting: config.getEnv('endpoints.formWaiting'), - endpointWebhook: config.getEnv('endpoints.webhook'), - endpointWebhookTest: config.getEnv('endpoints.webhookTest'), + endpointForm: this.globalConfig.endpoints.form, + endpointFormTest: this.globalConfig.endpoints.formTest, + endpointFormWaiting: this.globalConfig.endpoints.formWaiting, + endpointWebhook: this.globalConfig.endpoints.webhook, + endpointWebhookTest: this.globalConfig.endpoints.webhookTest, saveDataErrorExecution: config.getEnv('executions.saveDataOnError'), saveDataSuccessExecution: config.getEnv('executions.saveDataOnSuccess'), saveManualExecutions: config.getEnv('executions.saveDataManualExecutions'), @@ -160,6 +160,9 @@ export class FrontendService { workflowTagsDisabled: config.getEnv('workflowTagsDisabled'), logLevel: config.getEnv('logs.level'), hiringBannerEnabled: config.getEnv('hiringBanner.enabled'), + aiAssistant: { + enabled: false, + }, templates: { enabled: this.globalConfig.templates.enabled, host: this.globalConfig.templates.host, @@ -244,9 +247,9 @@ export class FrontendService { } getSettings(pushRef?: string): IN8nUISettings { - void this.internalHooks.onFrontendSettingsAPI(pushRef); + this.eventService.emit('session-started', { pushRef }); - const restEndpoint = config.getEnv('endpoints.rest'); + const restEndpoint = this.globalConfig.endpoints.rest; // Update all urls, in case `WEBHOOK_URL` was updated by `--tunnel` const instanceBaseUrl = this.urlService.getInstanceBaseUrl(); @@ -279,6 +282,7 @@ export class FrontendService { const isS3Selected = config.getEnv('binaryDataManager.mode') === 's3'; const isS3Available = config.getEnv('binaryDataManager.availableModes').includes('s3'); const isS3Licensed = this.license.isBinaryDataS3Licensed(); + const isAiAssistantEnabled = this.license.isAiAssistantEnabled(); this.settings.license.planName = this.license.getPlanName(); this.settings.license.consumerId = this.license.getConsumerId(); @@ -331,6 +335,10 @@ export class FrontendService { this.settings.missingPackages = this.communityPackagesService.hasMissingPackages; } + if (isAiAssistantEnabled) { + this.settings.aiAssistant.enabled = isAiAssistantEnabled; + } + this.settings.mfa.enabled = config.get('mfa.enabled'); this.settings.executionMode = config.getEnv('executions.mode'); diff --git a/packages/cli/src/services/import.service.ts b/packages/cli/src/services/import.service.ts index 96892e2745..7cf4a6493c 100644 --- a/packages/cli/src/services/import.service.ts +++ b/packages/cli/src/services/import.service.ts @@ -2,12 +2,12 @@ import { Service } from 'typedi'; import { v4 as uuid } from 'uuid'; import { type INode, type INodeCredentialsDetails } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import * as Db from '@/Db'; import { CredentialsRepository } from '@db/repositories/credentials.repository'; import { TagRepository } from '@db/repositories/tag.repository'; import { SharedWorkflow } from '@db/entities/SharedWorkflow'; -import { replaceInvalidCredentials } from '@/WorkflowHelpers'; +import { replaceInvalidCredentials } from '@/workflow-helpers'; import { Project } from '@db/entities/Project'; import { WorkflowEntity } from '@db/entities/WorkflowEntity'; import { WorkflowTagMapping } from '@db/entities/WorkflowTagMapping'; diff --git a/packages/cli/src/services/jwt.service.ts b/packages/cli/src/services/jwt.service.ts index 7f9f4fdd84..bcc2cdcbad 100644 --- a/packages/cli/src/services/jwt.service.ts +++ b/packages/cli/src/services/jwt.service.ts @@ -23,11 +23,15 @@ export class JwtService { } } - public sign(payload: object, options: jwt.SignOptions = {}): string { + sign(payload: object, options: jwt.SignOptions = {}): string { return jwt.sign(payload, this.jwtSecret, options); } - public verify(token: string, options: jwt.VerifyOptions = {}) { + decode(token: string) { + return jwt.decode(token) as JwtPayload; + } + + verify(token: string, options: jwt.VerifyOptions = {}) { return jwt.verify(token, this.jwtSecret, options) as T; } } diff --git a/packages/cli/src/services/orchestration.handler.base.service.ts b/packages/cli/src/services/orchestration.handler.base.service.ts index 6a706434e4..b2507151ce 100644 --- a/packages/cli/src/services/orchestration.handler.base.service.ts +++ b/packages/cli/src/services/orchestration.handler.base.service.ts @@ -2,6 +2,7 @@ import Container from 'typedi'; import { RedisService } from './redis.service'; import type { RedisServicePubSubSubscriber } from './redis/RedisServicePubSubSubscriber'; import type { WorkerCommandReceivedHandlerOptions } from './orchestration/worker/types'; +import type { MainResponseReceivedHandlerOptions } from './orchestration/main/types'; export abstract class OrchestrationHandlerService { protected initialized = false; @@ -19,7 +20,9 @@ export abstract class OrchestrationHandlerService { this.initialized = true; } - async initWithOptions(options: WorkerCommandReceivedHandlerOptions) { + async initWithOptions( + options: WorkerCommandReceivedHandlerOptions | MainResponseReceivedHandlerOptions, + ) { await this.initSubscriber(options); this.initialized = true; } @@ -29,5 +32,7 @@ export abstract class OrchestrationHandlerService { this.initialized = false; } - protected abstract initSubscriber(options?: WorkerCommandReceivedHandlerOptions): Promise; + protected abstract initSubscriber( + options?: WorkerCommandReceivedHandlerOptions | MainResponseReceivedHandlerOptions, + ): Promise; } diff --git a/packages/cli/src/services/orchestration.service.ts b/packages/cli/src/services/orchestration.service.ts index 3d74f187ac..f9bfc13728 100644 --- a/packages/cli/src/services/orchestration.service.ts +++ b/packages/cli/src/services/orchestration.service.ts @@ -1,5 +1,5 @@ import { Service } from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import config from '@/config'; import type { RedisServicePubSubPublisher } from './redis/RedisServicePubSubPublisher'; import type { RedisServiceBaseCommand, RedisServiceCommand } from './redis/RedisServiceCommands'; @@ -7,11 +7,13 @@ import type { RedisServiceBaseCommand, RedisServiceCommand } from './redis/Redis import { RedisService } from './redis.service'; import { MultiMainSetup } from './orchestration/main/MultiMainSetup.ee'; import type { WorkflowActivateMode } from 'n8n-workflow'; +import { InstanceSettings } from 'n8n-core'; @Service() export class OrchestrationService { constructor( private readonly logger: Logger, + private readonly instanceSettings: InstanceSettings, private readonly redisService: RedisService, readonly multiMainSetup: MultiMainSetup, ) {} @@ -43,15 +45,14 @@ export class OrchestrationService { return config.getEnv('redis.queueModeId'); } - /** - * Whether this instance is the leader in a multi-main setup. Always `false` in single-main setup. - */ + /** @deprecated use InstanceSettings.isLeader */ get isLeader() { - return config.getEnv('multiMainSetup.instanceType') === 'leader'; + return this.instanceSettings.isLeader; } + /** @deprecated use InstanceSettings.isFollower */ get isFollower() { - return config.getEnv('multiMainSetup.instanceType') !== 'leader'; + return this.instanceSettings.isFollower; } sanityCheck() { @@ -66,7 +67,7 @@ export class OrchestrationService { if (this.isMultiMainSetupEnabled) { await this.multiMainSetup.init(); } else { - config.set('multiMainSetup.instanceType', 'leader'); + this.instanceSettings.markAsLeader(); } this.isInitialized = true; diff --git a/packages/cli/src/services/orchestration/helpers.ts b/packages/cli/src/services/orchestration/helpers.ts index c5ccd43636..262c36ccee 100644 --- a/packages/cli/src/services/orchestration/helpers.ts +++ b/packages/cli/src/services/orchestration/helpers.ts @@ -1,6 +1,6 @@ import { Container } from 'typedi'; import { jsonParse } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { RedisServiceCommandObject } from '../redis/RedisServiceCommands'; import { COMMAND_REDIS_CHANNEL } from '../redis/RedisConstants'; import * as os from 'os'; diff --git a/packages/cli/src/services/orchestration/main/MultiMainSetup.ee.ts b/packages/cli/src/services/orchestration/main/MultiMainSetup.ee.ts index 19cd14c4a8..82c28d61f3 100644 --- a/packages/cli/src/services/orchestration/main/MultiMainSetup.ee.ts +++ b/packages/cli/src/services/orchestration/main/MultiMainSetup.ee.ts @@ -1,16 +1,23 @@ -import { EventEmitter } from 'node:events'; import config from '@/config'; import { Service } from 'typedi'; import { TIME } from '@/constants'; +import { InstanceSettings } from 'n8n-core'; import { ErrorReporterProxy as EventReporter } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { RedisServicePubSubPublisher } from '@/services/redis/RedisServicePubSubPublisher'; import { RedisClientService } from '@/services/redis/redis-client.service'; +import { TypedEmitter } from '@/TypedEmitter'; + +type MultiMainEvents = { + 'leader-stepdown': never; + 'leader-takeover': never; +}; @Service() -export class MultiMainSetup extends EventEmitter { +export class MultiMainSetup extends TypedEmitter { constructor( private readonly logger: Logger, + private readonly instanceSettings: InstanceSettings, private readonly redisPublisher: RedisServicePubSubPublisher, private readonly redisClientService: RedisClientService, ) { @@ -45,7 +52,7 @@ export class MultiMainSetup extends EventEmitter { async shutdown() { clearInterval(this.leaderCheckInterval); - const isLeader = config.getEnv('multiMainSetup.instanceType') === 'leader'; + const { isLeader } = this.instanceSettings; if (isLeader) await this.redisPublisher.clear(this.leaderKey); } @@ -64,8 +71,8 @@ export class MultiMainSetup extends EventEmitter { if (leaderId && leaderId !== this.instanceId) { this.logger.debug(`[Instance ID ${this.instanceId}] Leader is other instance "${leaderId}"`); - if (config.getEnv('multiMainSetup.instanceType') === 'leader') { - config.set('multiMainSetup.instanceType', 'follower'); + if (this.instanceSettings.isLeader) { + this.instanceSettings.markAsFollower(); this.emit('leader-stepdown'); // lost leadership - stop triggers, pollers, pruning, wait-tracking, queue recovery @@ -80,7 +87,7 @@ export class MultiMainSetup extends EventEmitter { `[Instance ID ${this.instanceId}] Leadership vacant, attempting to become leader...`, ); - config.set('multiMainSetup.instanceType', 'follower'); + this.instanceSettings.markAsFollower(); /** * Lost leadership - stop triggers, pollers, pruning, wait tracking, license renewal, queue recovery @@ -101,7 +108,7 @@ export class MultiMainSetup extends EventEmitter { if (keySetSuccessfully) { this.logger.debug(`[Instance ID ${this.instanceId}] Leader is now this instance`); - config.set('multiMainSetup.instanceType', 'leader'); + this.instanceSettings.markAsLeader(); await this.redisPublisher.setExpiration(this.leaderKey, this.leaderKeyTtl); @@ -110,7 +117,7 @@ export class MultiMainSetup extends EventEmitter { */ this.emit('leader-takeover'); } else { - config.set('multiMainSetup.instanceType', 'follower'); + this.instanceSettings.markAsFollower(); } } diff --git a/packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts b/packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts index 8abcbe78b2..1cdb3cf7f1 100644 --- a/packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts +++ b/packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts @@ -2,14 +2,15 @@ import { Container } from 'typedi'; import { debounceMessageReceiver, messageToRedisServiceCommandObject } from '../helpers'; import config from '@/config'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { License } from '@/License'; -import { Logger } from '@/Logger'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; +import { License } from '@/license'; +import { Logger } from '@/logger'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; import { Push } from '@/push'; -import { TestWebhooks } from '@/TestWebhooks'; +import { TestWebhooks } from '@/webhooks/test-webhooks'; import { OrchestrationService } from '@/services/orchestration.service'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import { CommunityPackagesService } from '@/services/communityPackages.service'; // eslint-disable-next-line complexity export async function handleCommandMessageMain(messageString: string) { @@ -77,6 +78,20 @@ export async function handleCommandMessageMain(messageString: string) { } await Container.get(ExternalSecretsManager).reloadAllProviders(); break; + case 'community-package-install': + case 'community-package-update': + case 'community-package-uninstall': + if (!debounceMessageReceiver(message, 200)) { + return message; + } + const { packageName, packageVersion } = message.payload; + const communityPackagesService = Container.get(CommunityPackagesService); + if (message.command === 'community-package-uninstall') { + await communityPackagesService.removeNpmPackage(packageName); + } else { + await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion); + } + break; case 'add-webhooks-triggers-and-pollers': { if (!debounceMessageReceiver(message, 100)) { diff --git a/packages/cli/src/services/orchestration/main/handleWorkerResponseMessageMain.ts b/packages/cli/src/services/orchestration/main/handleWorkerResponseMessageMain.ts index 47a1a08019..b623f0b4f2 100644 --- a/packages/cli/src/services/orchestration/main/handleWorkerResponseMessageMain.ts +++ b/packages/cli/src/services/orchestration/main/handleWorkerResponseMessageMain.ts @@ -1,27 +1,42 @@ import { jsonParse } from 'n8n-workflow'; import Container from 'typedi'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { Push } from '../../../push'; import type { RedisServiceWorkerResponseObject } from '../../redis/RedisServiceCommands'; +import { WORKER_RESPONSE_REDIS_CHANNEL } from '@/services/redis/RedisConstants'; +import type { MainResponseReceivedHandlerOptions } from './types'; -export async function handleWorkerResponseMessageMain(messageString: string) { - const workerResponse = jsonParse(messageString); - if (workerResponse) { - switch (workerResponse.command) { - case 'getStatus': - const push = Container.get(Push); - push.broadcast('sendWorkerStatusMessage', { - workerId: workerResponse.workerId, - status: workerResponse.payload, - }); - break; - case 'getId': - break; - default: - Container.get(Logger).debug( - `Received worker response ${workerResponse.command} from ${workerResponse.workerId}`, - ); - } +export async function handleWorkerResponseMessageMain( + messageString: string, + options: MainResponseReceivedHandlerOptions, +) { + const workerResponse = jsonParse(messageString, { + fallbackValue: null, + }); + + if (!workerResponse) { + Container.get(Logger).debug( + `Received invalid message via channel ${WORKER_RESPONSE_REDIS_CHANNEL}: "${messageString}"`, + ); + return; } + + if (workerResponse.targets && !workerResponse.targets.includes(options.queueModeId)) return; + + switch (workerResponse.command) { + case 'getStatus': + Container.get(Push).broadcast('sendWorkerStatusMessage', { + workerId: workerResponse.workerId, + status: workerResponse.payload, + }); + break; + case 'getId': + break; + default: + Container.get(Logger).debug( + `Received worker response ${workerResponse.command} from ${workerResponse.workerId}`, + ); + } + return workerResponse; } diff --git a/packages/cli/src/services/orchestration/main/orchestration.handler.main.service.ts b/packages/cli/src/services/orchestration/main/orchestration.handler.main.service.ts index 6cc86c9f51..983c39fd33 100644 --- a/packages/cli/src/services/orchestration/main/orchestration.handler.main.service.ts +++ b/packages/cli/src/services/orchestration/main/orchestration.handler.main.service.ts @@ -3,10 +3,11 @@ import { COMMAND_REDIS_CHANNEL, WORKER_RESPONSE_REDIS_CHANNEL } from '../../redi import { handleWorkerResponseMessageMain } from './handleWorkerResponseMessageMain'; import { handleCommandMessageMain } from './handleCommandMessageMain'; import { OrchestrationHandlerService } from '../../orchestration.handler.base.service'; +import type { MainResponseReceivedHandlerOptions } from './types'; @Service() export class OrchestrationHandlerMainService extends OrchestrationHandlerService { - async initSubscriber() { + async initSubscriber(options: MainResponseReceivedHandlerOptions) { this.redisSubscriber = await this.redisService.getPubSubSubscriber(); await this.redisSubscriber.subscribeToCommandChannel(); @@ -16,7 +17,7 @@ export class OrchestrationHandlerMainService extends OrchestrationHandlerService 'OrchestrationMessageReceiver', async (channel: string, messageString: string) => { if (channel === WORKER_RESPONSE_REDIS_CHANNEL) { - await handleWorkerResponseMessageMain(messageString); + await handleWorkerResponseMessageMain(messageString, options); } else if (channel === COMMAND_REDIS_CHANNEL) { await handleCommandMessageMain(messageString); } diff --git a/packages/cli/src/services/orchestration/main/types.ts b/packages/cli/src/services/orchestration/main/types.ts new file mode 100644 index 0000000000..d189d7cdf7 --- /dev/null +++ b/packages/cli/src/services/orchestration/main/types.ts @@ -0,0 +1,6 @@ +import type { RedisServicePubSubPublisher } from '@/services/redis/RedisServicePubSubPublisher'; + +export type MainResponseReceivedHandlerOptions = { + queueModeId: string; + redisPublisher: RedisServicePubSubPublisher; +}; diff --git a/packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts b/packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts index 9dc326978d..c6f746db63 100644 --- a/packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts +++ b/packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts @@ -1,10 +1,11 @@ -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { License } from '@/License'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; +import { License } from '@/license'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; import Container from 'typedi'; import { Logger } from 'winston'; import { messageToRedisServiceCommandObject, debounceMessageReceiver } from '../helpers'; import config from '@/config'; +import { CommunityPackagesService } from '@/services/communityPackages.service'; export async function handleCommandMessageWebhook(messageString: string) { const queueModeId = config.getEnv('redis.queueModeId'); @@ -63,6 +64,20 @@ export async function handleCommandMessageWebhook(messageString: string) { } await Container.get(ExternalSecretsManager).reloadAllProviders(); break; + case 'community-package-install': + case 'community-package-update': + case 'community-package-uninstall': + if (!debounceMessageReceiver(message, 200)) { + return message; + } + const { packageName, packageVersion } = message.payload; + const communityPackagesService = Container.get(CommunityPackagesService); + if (message.command === 'community-package-uninstall') { + await communityPackagesService.removeNpmPackage(packageName); + } else { + await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion); + } + break; default: break; diff --git a/packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts b/packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts index fa9ee67675..52727c37e1 100644 --- a/packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts +++ b/packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts @@ -3,13 +3,14 @@ import Container from 'typedi'; import type { RedisServiceCommandObject } from '@/services/redis/RedisServiceCommands'; import { COMMAND_REDIS_CHANNEL } from '@/services/redis/RedisConstants'; import * as os from 'os'; -import { License } from '@/License'; +import { License } from '@/license'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; +import { ExternalSecretsManager } from '@/external-secrets/external-secrets-manager.ee'; import { debounceMessageReceiver, getOsCpuString } from '../helpers'; import type { WorkerCommandReceivedHandlerOptions } from './types'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { N8N_VERSION } from '@/constants'; +import { CommunityPackagesService } from '@/services/communityPackages.service'; export function getWorkerCommandReceivedHandler(options: WorkerCommandReceivedHandlerOptions) { // eslint-disable-next-line complexity @@ -112,6 +113,18 @@ export function getWorkerCommandReceivedHandler(options: WorkerCommandReceivedHa }); } break; + case 'community-package-install': + case 'community-package-update': + case 'community-package-uninstall': + if (!debounceMessageReceiver(message, 500)) return; + const { packageName, packageVersion } = message.payload; + const communityPackagesService = Container.get(CommunityPackagesService); + if (message.command === 'community-package-uninstall') { + await communityPackagesService.removeNpmPackage(packageName); + } else { + await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion); + } + break; case 'reloadLicense': if (!debounceMessageReceiver(message, 500)) return; await Container.get(License).reload(); diff --git a/packages/cli/src/services/orchestration/worker/types.ts b/packages/cli/src/services/orchestration/worker/types.ts index 351c56394a..84c515466e 100644 --- a/packages/cli/src/services/orchestration/worker/types.ts +++ b/packages/cli/src/services/orchestration/worker/types.ts @@ -1,11 +1,12 @@ import type { ExecutionStatus, WorkflowExecuteMode } from 'n8n-workflow'; import type { RedisServicePubSubPublisher } from '../../redis/RedisServicePubSubPublisher'; +import type { RunningJobSummary } from '@/scaling/types'; export interface WorkerCommandReceivedHandlerOptions { queueModeId: string; redisPublisher: RedisServicePubSubPublisher; - getRunningJobIds: () => string[]; - getRunningJobsSummary: () => WorkerJobStatusSummary[]; + getRunningJobIds: () => Array; + getRunningJobsSummary: () => RunningJobSummary[]; } export interface WorkerJobStatusSummary { diff --git a/packages/cli/src/services/project.service.ts b/packages/cli/src/services/project.service.ts index 8df30da1e9..6905019017 100644 --- a/packages/cli/src/services/project.service.ts +++ b/packages/cli/src/services/project.service.ts @@ -17,7 +17,7 @@ import { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflo import { SharedCredentialsRepository } from '@/databases/repositories/sharedCredentials.repository'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { CacheService } from './cache/cache.service'; -import { License } from '@/License'; +import { License } from '@/license'; import { UNLIMITED_LICENSE_QUOTA } from '@/constants'; import { ApplicationError } from 'n8n-workflow'; diff --git a/packages/cli/src/services/pruning.service.ts b/packages/cli/src/services/pruning.service.ts index daa0883e1f..dab4bbf8e8 100644 --- a/packages/cli/src/services/pruning.service.ts +++ b/packages/cli/src/services/pruning.service.ts @@ -1,11 +1,11 @@ import { Service } from 'typedi'; -import { BinaryDataService } from 'n8n-core'; +import { BinaryDataService, InstanceSettings } from 'n8n-core'; import { inTest, TIME } from '@/constants'; import config from '@/config'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { jsonStringify } from 'n8n-workflow'; -import { OnShutdown } from '@/decorators/OnShutdown'; +import { OnShutdown } from '@/decorators/on-shutdown'; import { OrchestrationService } from './orchestration.service'; @Service() @@ -25,6 +25,7 @@ export class PruningService { constructor( private readonly logger: Logger, + private readonly instanceSettings: InstanceSettings, private readonly executionRepository: ExecutionRepository, private readonly binaryDataService: BinaryDataService, private readonly orchestrationService: OrchestrationService, @@ -56,7 +57,7 @@ export class PruningService { if ( config.getEnv('multiMainSetup.enabled') && config.getEnv('generic.instanceType') === 'main' && - config.getEnv('multiMainSetup.instanceType') === 'follower' + this.instanceSettings.isFollower ) { return false; } diff --git a/packages/cli/src/services/redis/RedisConstants.ts b/packages/cli/src/services/redis/RedisConstants.ts index 281817b9c7..038e94e9ce 100644 --- a/packages/cli/src/services/redis/RedisConstants.ts +++ b/packages/cli/src/services/redis/RedisConstants.ts @@ -1,6 +1,2 @@ -export const EVENT_BUS_REDIS_STREAM = 'n8n:eventstream'; -export const COMMAND_REDIS_STREAM = 'n8n:commandstream'; -export const WORKER_RESPONSE_REDIS_STREAM = 'n8n:workerstream'; export const COMMAND_REDIS_CHANNEL = 'n8n.commands'; export const WORKER_RESPONSE_REDIS_CHANNEL = 'n8n.worker-response'; -export const WORKER_RESPONSE_REDIS_LIST = 'n8n:list:worker-response'; diff --git a/packages/cli/src/services/redis/RedisServiceBaseClasses.ts b/packages/cli/src/services/redis/RedisServiceBaseClasses.ts index ba2cc41a91..d31044bb5c 100644 --- a/packages/cli/src/services/redis/RedisServiceBaseClasses.ts +++ b/packages/cli/src/services/redis/RedisServiceBaseClasses.ts @@ -2,22 +2,9 @@ import type Redis from 'ioredis'; import type { Cluster } from 'ioredis'; import { Service } from 'typedi'; import config from '@/config'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { RedisClientService } from './redis-client.service'; - -export type RedisClientType = - | 'subscriber' - | 'client' - | 'bclient' - | 'subscriber(bull)' - | 'client(bull)' - | 'bclient(bull)' - | 'client(cache)' - | 'publisher' - | 'consumer' - | 'producer' - | 'list-sender' - | 'list-receiver'; +import type { RedisClientType } from './redis.types'; export type RedisServiceMessageHandler = | ((channel: string, message: string) => void) @@ -34,7 +21,7 @@ class RedisServiceBase { private readonly redisClientService: RedisClientService, ) {} - async init(type: RedisClientType = 'client'): Promise { + async init(type: RedisClientType): Promise { if (this.redisClient && this.isInitialized) { return; } @@ -45,6 +32,8 @@ class RedisServiceBase { this.logger.warn('Error with Redis: ', error); } }); + + this.isInitialized = true; } async destroy(): Promise { @@ -60,7 +49,7 @@ class RedisServiceBase { export abstract class RedisServiceBaseSender extends RedisServiceBase { senderId: string; - async init(type: RedisClientType = 'client'): Promise { + async init(type: RedisClientType): Promise { await super.init(type); this.senderId = config.get('redis.queueModeId'); } diff --git a/packages/cli/src/services/redis/RedisServiceCommands.ts b/packages/cli/src/services/redis/RedisServiceCommands.ts index 009f39ef65..b7786adad3 100644 --- a/packages/cli/src/services/redis/RedisServiceCommands.ts +++ b/packages/cli/src/services/redis/RedisServiceCommands.ts @@ -7,6 +7,9 @@ export type RedisServiceCommand = | 'stopWorker' | 'reloadLicense' | 'reloadExternalSecretsProviders' + | 'community-package-install' + | 'community-package-update' + | 'community-package-uninstall' | 'display-workflow-activation' // multi-main only | 'display-workflow-deactivation' // multi-main only | 'add-webhooks-triggers-and-pollers' // multi-main only @@ -26,7 +29,11 @@ export type RedisServiceBaseCommand = senderId: string; command: Exclude< RedisServiceCommand, - 'relay-execution-lifecycle-event' | 'clear-test-webhooks' + | 'relay-execution-lifecycle-event' + | 'clear-test-webhooks' + | 'community-package-install' + | 'community-package-update' + | 'community-package-uninstall' >; payload?: { [key: string]: string | number | boolean | string[] | number[] | boolean[]; @@ -41,6 +48,14 @@ export type RedisServiceBaseCommand = senderId: string; command: 'clear-test-webhooks'; payload: { webhookKey: string; workflowEntity: IWorkflowDb; pushRef: string }; + } + | { + senderId: string; + command: + | 'community-package-install' + | 'community-package-update' + | 'community-package-uninstall'; + payload: { packageName: string; packageVersion: string }; }; export type RedisServiceWorkerResponseObject = { @@ -79,7 +94,7 @@ export type RedisServiceWorkerResponseObject = { workflowId: string; }; } -); +) & { targets?: string[] }; export type RedisServiceCommandObject = { targets?: string[]; diff --git a/packages/cli/src/services/redis/RedisServicePubSubPublisher.ts b/packages/cli/src/services/redis/RedisServicePubSubPublisher.ts index ff810ba7c6..23ca9a5b69 100644 --- a/packages/cli/src/services/redis/RedisServicePubSubPublisher.ts +++ b/packages/cli/src/services/redis/RedisServicePubSubPublisher.ts @@ -9,7 +9,7 @@ import { RedisServiceBaseSender } from './RedisServiceBaseClasses'; @Service() export class RedisServicePubSubPublisher extends RedisServiceBaseSender { async init(): Promise { - await super.init('publisher'); + await super.init('publisher(n8n)'); } async publish(channel: string, message: string): Promise { diff --git a/packages/cli/src/services/redis/RedisServicePubSubSubscriber.ts b/packages/cli/src/services/redis/RedisServicePubSubSubscriber.ts index 8751826428..144647009f 100644 --- a/packages/cli/src/services/redis/RedisServicePubSubSubscriber.ts +++ b/packages/cli/src/services/redis/RedisServicePubSubSubscriber.ts @@ -5,7 +5,7 @@ import { RedisServiceBaseReceiver } from './RedisServiceBaseClasses'; @Service() export class RedisServicePubSubSubscriber extends RedisServiceBaseReceiver { async init(): Promise { - await super.init('subscriber'); + await super.init('subscriber(n8n)'); this.redisClient?.on('message', (channel: string, message: string) => { this.messageHandlers.forEach((handler: (channel: string, message: string) => void) => diff --git a/packages/cli/src/services/redis/redis-client.service.ts b/packages/cli/src/services/redis/redis-client.service.ts index 7363a9c9b7..bf70d8114d 100644 --- a/packages/cli/src/services/redis/redis-client.service.ts +++ b/packages/cli/src/services/redis/redis-client.service.ts @@ -1,17 +1,21 @@ import { Service } from 'typedi'; -import config from '@/config'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import ioRedis from 'ioredis'; import type { Cluster, RedisOptions } from 'ioredis'; -import type { RedisClientType } from './RedisServiceBaseClasses'; -import { OnShutdown } from '@/decorators/OnShutdown'; +import type { RedisClientType } from './redis.types'; + +import { OnShutdown } from '@/decorators/on-shutdown'; import { LOWEST_SHUTDOWN_PRIORITY } from '@/constants'; +import { GlobalConfig } from '@n8n/config'; @Service() export class RedisClientService { private readonly clients = new Set(); - constructor(private readonly logger: Logger) {} + constructor( + private readonly logger: Logger, + private readonly globalConfig: GlobalConfig, + ) {} createClient(arg: { type: RedisClientType; extraOptions?: RedisOptions }) { const client = @@ -57,7 +61,7 @@ export class RedisClientService { }) { const options = this.getOptions({ extraOptions }); - const { host, port } = config.getEnv('queue.bull.redis'); + const { host, port } = this.globalConfig.queue.bull.redis; options.host = host; options.port = port; @@ -87,7 +91,7 @@ export class RedisClientService { } private getOptions({ extraOptions }: { extraOptions?: RedisOptions }) { - const { username, password, db, tls } = config.getEnv('queue.bull.redis'); + const { username, password, db, tls } = this.globalConfig.queue.bull.redis; /** * Disabling ready check allows quick reconnection to Redis if Redis becomes @@ -124,7 +128,7 @@ export class RedisClientService { private retryStrategy() { const RETRY_INTERVAL = 500; // ms const RESET_LENGTH = 30_000; // ms - const MAX_TIMEOUT = config.getEnv('queue.bull.redis.timeoutThreshold'); + const MAX_TIMEOUT = this.globalConfig.queue.bull.redis.timeoutThreshold; let lastAttemptTs = 0; let cumulativeTimeout = 0; @@ -152,8 +156,7 @@ export class RedisClientService { } private clusterNodes() { - return config - .getEnv('queue.bull.redis.clusterNodes') + return this.globalConfig.queue.bull.redis.clusterNodes .split(',') .filter((pair) => pair.trim().length > 0) .map((pair) => { diff --git a/packages/cli/src/services/redis/redis.types.ts b/packages/cli/src/services/redis/redis.types.ts new file mode 100644 index 0000000000..ed694904d7 --- /dev/null +++ b/packages/cli/src/services/redis/redis.types.ts @@ -0,0 +1,19 @@ +export type RedisClientType = N8nRedisClientType | BullRedisClientType; + +/** + * Redis client used by n8n. + * + * - `subscriber(n8n)` to listen for messages from scaling mode communication channels + * - `publisher(n8n)` to send messages into scaling mode communication channels + * - `cache(n8n)` for caching operations (variables, resource ownership, etc.) + */ +type N8nRedisClientType = 'subscriber(n8n)' | 'publisher(n8n)' | 'cache(n8n)'; + +/** + * Redis client used internally by Bull. Suffixed with `(bull)` at `ScalingService.setupQueue`. + * + * - `subscriber(bull)` for event listening + * - `client(bull)` for general queue operations + * - `bclient(bull)` for blocking operations when processing jobs + */ +type BullRedisClientType = 'subscriber(bull)' | 'client(bull)' | 'bclient(bull)'; diff --git a/packages/cli/src/services/role.service.ts b/packages/cli/src/services/role.service.ts index 292db6dfa3..02631b20f0 100644 --- a/packages/cli/src/services/role.service.ts +++ b/packages/cli/src/services/role.service.ts @@ -26,7 +26,7 @@ import type { ListQuery } from '@/requests'; import { combineScopes, type Resource, type Scope } from '@n8n/permissions'; import { Service } from 'typedi'; import { ApplicationError } from 'n8n-workflow'; -import { License } from '@/License'; +import { License } from '@/license'; import type { CredentialsEntity } from '@/databases/entities/CredentialsEntity'; export type RoleNamespace = 'global' | 'project' | 'credential' | 'workflow'; diff --git a/packages/cli/src/services/tag.service.ts b/packages/cli/src/services/tag.service.ts index 27f4ebd0f2..141749cbf8 100644 --- a/packages/cli/src/services/tag.service.ts +++ b/packages/cli/src/services/tag.service.ts @@ -1,9 +1,9 @@ import { TagRepository } from '@db/repositories/tag.repository'; import { Service } from 'typedi'; -import { validateEntity } from '@/GenericHelpers'; +import { validateEntity } from '@/generic-helpers'; import type { ITagWithCountDb } from '@/Interfaces'; import type { TagEntity } from '@db/entities/TagEntity'; -import { ExternalHooks } from '@/ExternalHooks'; +import { ExternalHooks } from '@/external-hooks'; type GetAllResult = T extends { withUsageCount: true } ? ITagWithCountDb[] : TagEntity[]; diff --git a/packages/cli/src/services/url.service.ts b/packages/cli/src/services/url.service.ts index b1ef737c2f..8c0e1948fc 100644 --- a/packages/cli/src/services/url.service.ts +++ b/packages/cli/src/services/url.service.ts @@ -1,12 +1,13 @@ import { Service } from 'typedi'; import config from '@/config'; +import { GlobalConfig } from '@n8n/config'; @Service() export class UrlService { /** Returns the base URL n8n is reachable from */ readonly baseUrl: string; - constructor() { + constructor(private readonly globalConfig: GlobalConfig) { this.baseUrl = this.generateBaseUrl(); } @@ -27,10 +28,7 @@ export class UrlService { } private generateBaseUrl(): string { - const protocol = config.getEnv('protocol'); - const host = config.getEnv('host'); - const port = config.getEnv('port'); - const path = config.getEnv('path'); + const { path, port, host, protocol } = this.globalConfig; if ((protocol === 'http' && port === 80) || (protocol === 'https' && port === 443)) { return `${protocol}://${host}${path}`; diff --git a/packages/cli/src/services/user.service.ts b/packages/cli/src/services/user.service.ts index cc50e09f74..28bbe3d8a2 100644 --- a/packages/cli/src/services/user.service.ts +++ b/packages/cli/src/services/user.service.ts @@ -1,4 +1,4 @@ -import { Container, Service } from 'typedi'; +import { Service } from 'typedi'; import type { IUserSettings } from 'n8n-workflow'; import { ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; @@ -6,13 +6,12 @@ import type { User, AssignableRole } from '@db/entities/User'; import { UserRepository } from '@db/repositories/user.repository'; import type { Invitation, PublicUser } from '@/Interfaces'; import type { PostHogClient } from '@/posthog'; -import { Logger } from '@/Logger'; -import { UserManagementMailer } from '@/UserManagement/email'; -import { InternalHooks } from '@/InternalHooks'; +import { Logger } from '@/logger'; +import { UserManagementMailer } from '@/user-management/email'; import { UrlService } from '@/services/url.service'; import type { UserRequest } from '@/requests'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @Service() export class UserService { @@ -144,32 +143,28 @@ export class UserService { if (result.emailSent) { invitedUser.user.emailSent = true; delete invitedUser.user?.inviteAcceptUrl; - void Container.get(InternalHooks).onUserTransactionalEmail({ - user_id: id, - message_type: 'New user invite', - public_api: false, + + this.eventService.emit('user-transactional-email-sent', { + userId: id, + messageType: 'New user invite', + publicApi: false, }); } - void Container.get(InternalHooks).onUserInvite({ - user: owner, - target_user_id: Object.values(toInviteUsers), - public_api: false, - email_sent: result.emailSent, - invitee_role: role, // same role for all invited users - }); this.eventService.emit('user-invited', { user: owner, targetUserId: Object.values(toInviteUsers), + publicApi: false, + emailSent: result.emailSent, + inviteeRole: role, // same role for all invited users }); } catch (e) { if (e instanceof Error) { - void Container.get(InternalHooks).onEmailFailed({ + this.eventService.emit('email-failed', { user: owner, - message_type: 'New user invite', - public_api: false, + messageType: 'New user invite', + publicApi: false, }); - this.eventService.emit('email-failed', { user: owner, messageType: 'New user invite' }); this.logger.error('Failed to send email', { userId: owner.id, inviteAcceptUrl, diff --git a/packages/cli/src/services/workflow-statistics.service.ts b/packages/cli/src/services/workflow-statistics.service.ts index 516732add8..a58b6ffbbf 100644 --- a/packages/cli/src/services/workflow-statistics.service.ts +++ b/packages/cli/src/services/workflow-statistics.service.ts @@ -1,29 +1,50 @@ -import { EventEmitter } from 'events'; -import { Container, Service } from 'typedi'; +import { Service } from 'typedi'; import type { INode, IRun, IWorkflowBase } from 'n8n-workflow'; import { StatisticsNames } from '@db/entities/WorkflowStatistics'; import { WorkflowStatisticsRepository } from '@db/repositories/workflowStatistics.repository'; import { UserService } from '@/services/user.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { OwnershipService } from './ownership.service'; +import { TypedEmitter } from '@/TypedEmitter'; +import { EventService } from '@/events/event.service'; + +type WorkflowStatisticsEvents = { + nodeFetchedData: { workflowId: string; node: INode }; + workflowExecutionCompleted: { workflowData: IWorkflowBase; fullRunData: IRun }; + 'telemetry.onFirstProductionWorkflowSuccess': { + project_id: string; + workflow_id: string; + user_id: string; + }; + 'telemetry.onFirstWorkflowDataLoad': { + user_id: string; + project_id: string; + workflow_id: string; + node_type: string; + node_id: string; + }; +}; @Service() -export class WorkflowStatisticsService extends EventEmitter { +export class WorkflowStatisticsService extends TypedEmitter { constructor( private readonly logger: Logger, private readonly repository: WorkflowStatisticsRepository, private readonly ownershipService: OwnershipService, + private readonly userService: UserService, + private readonly eventService: EventService, ) { super({ captureRejections: true }); if ('SKIP_STATISTICS_EVENTS' in process.env) return; this.on( 'nodeFetchedData', - async (workflowId, node) => await this.nodeFetchedData(workflowId, node), + async ({ workflowId, node }) => await this.nodeFetchedData(workflowId, node), ); this.on( 'workflowExecutionCompleted', - async (workflowData, runData) => await this.workflowExecutionCompleted(workflowData, runData), + async ({ workflowData, fullRunData }) => + await this.workflowExecutionCompleted(workflowData, fullRunData), ); } @@ -49,26 +70,23 @@ export class WorkflowStatisticsService extends EventEmitter { const upsertResult = await this.repository.upsertWorkflowStatistics(name, workflowId); if (name === StatisticsNames.productionSuccess && upsertResult === 'insert') { - const project = await Container.get(OwnershipService).getWorkflowProjectCached(workflowId); + const project = await this.ownershipService.getWorkflowProjectCached(workflowId); if (project.type === 'personal') { - const owner = await Container.get(OwnershipService).getProjectOwnerCached(project.id); - - const metrics = { - project_id: project.id, - workflow_id: workflowId, - user_id: owner?.id, - }; + const owner = await this.ownershipService.getProjectOwnerCached(project.id); if (owner && !owner.settings?.userActivated) { - await Container.get(UserService).updateSettings(owner.id, { + await this.userService.updateSettings(owner.id, { firstSuccessfulWorkflowId: workflowId, userActivated: true, userActivatedAt: runData.startedAt.getTime(), }); } - // Send the metrics - this.emit('telemetry.onFirstProductionWorkflowSuccess', metrics); + this.eventService.emit('first-production-workflow-succeeded', { + projectId: project.id, + workflowId, + userId: owner!.id, + }); } } } catch (error) { @@ -90,50 +108,23 @@ export class WorkflowStatisticsService extends EventEmitter { const owner = await this.ownershipService.getProjectOwnerCached(project.id); let metrics = { - user_id: owner?.id, - project_id: project.id, - workflow_id: workflowId, - node_type: node.type, - node_id: node.id, + userId: owner!.id, + project: project.id, + workflowId, + nodeType: node.type, + nodeId: node.id, }; // This is probably naive but I can't see a way for a node to have multiple credentials attached so.. if (node.credentials) { Object.entries(node.credentials).forEach(([credName, credDetails]) => { metrics = Object.assign(metrics, { - credential_type: credName, - credential_id: credDetails.id, + credentialType: credName, + credentialId: credDetails.id, }); }); } - // Send metrics to posthog - this.emit('telemetry.onFirstWorkflowDataLoad', metrics); + this.eventService.emit('first-workflow-data-loaded', metrics); } } - -export declare interface WorkflowStatisticsService { - on( - event: 'nodeFetchedData', - listener: (workflowId: string | undefined | null, node: INode) => void, - ): this; - on( - event: 'workflowExecutionCompleted', - listener: (workflowData: IWorkflowBase, runData: IRun) => void, - ): this; - on( - event: 'telemetry.onFirstProductionWorkflowSuccess', - listener: (metrics: { user_id: string; workflow_id: string }) => void, - ): this; - on( - event: 'telemetry.onFirstWorkflowDataLoad', - listener: (metrics: { - user_id: string; - workflow_id: string; - node_type: string; - node_id: string; - credential_type?: string; - credential_id?: string; - }) => void, - ): this; -} diff --git a/packages/cli/test/unit/shutdown/Shutdown.service.test.ts b/packages/cli/src/shutdown/__tests__/Shutdown.service.test.ts similarity index 97% rename from packages/cli/test/unit/shutdown/Shutdown.service.test.ts rename to packages/cli/src/shutdown/__tests__/Shutdown.service.test.ts index ca85f78564..16f37b3828 100644 --- a/packages/cli/test/unit/shutdown/Shutdown.service.test.ts +++ b/packages/cli/src/shutdown/__tests__/Shutdown.service.test.ts @@ -1,7 +1,7 @@ import { ApplicationError, ErrorReporterProxy } from 'n8n-workflow'; import { mock } from 'jest-mock-extended'; -import type { ServiceClass } from '@/shutdown/Shutdown.service'; -import { ShutdownService } from '@/shutdown/Shutdown.service'; +import type { ServiceClass } from '@/shutdown/shutdown.service'; +import { ShutdownService } from '@/shutdown/shutdown.service'; import Container from 'typedi'; class MockComponent { diff --git a/packages/cli/src/shutdown/Shutdown.service.ts b/packages/cli/src/shutdown/shutdown.service.ts similarity index 99% rename from packages/cli/src/shutdown/Shutdown.service.ts rename to packages/cli/src/shutdown/shutdown.service.ts index 772463f23a..f3927ec309 100644 --- a/packages/cli/src/shutdown/Shutdown.service.ts +++ b/packages/cli/src/shutdown/shutdown.service.ts @@ -1,7 +1,7 @@ import { Container, Service } from 'typedi'; import { ApplicationError, ErrorReporterProxy, assert } from 'n8n-workflow'; import type { Class } from 'n8n-core'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { LOWEST_SHUTDOWN_PRIORITY, HIGHEST_SHUTDOWN_PRIORITY } from '@/constants'; type HandlerFn = () => Promise | void; diff --git a/packages/cli/test/unit/sso/saml/saml.service.ee.test.ts b/packages/cli/src/sso/saml/__tests__/saml.service.ee.test.ts similarity index 93% rename from packages/cli/test/unit/sso/saml/saml.service.ee.test.ts rename to packages/cli/src/sso/saml/__tests__/saml.service.ee.test.ts index 9ba6ddaf2a..9821c89637 100644 --- a/packages/cli/test/unit/sso/saml/saml.service.ee.test.ts +++ b/packages/cli/src/sso/saml/__tests__/saml.service.ee.test.ts @@ -1,11 +1,11 @@ import { mock } from 'jest-mock-extended'; import type express from 'express'; import { SamlService } from '@/sso/saml/saml.service.ee'; -import { mockInstance } from '../../../shared/mocking'; +import { mockInstance } from '@test/mocking'; import { UrlService } from '@/services/url.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { IdentityProviderInstance, ServiceProviderInstance } from 'samlify'; -import * as samlHelpers from '@/sso/saml/samlHelpers'; +import * as samlHelpers from '@/sso/saml/saml-helpers'; describe('SamlService', () => { const logger = mockInstance(Logger); diff --git a/packages/cli/test/unit/sso/saml/samlHelpers.test.ts b/packages/cli/src/sso/saml/__tests__/samlHelpers.test.ts similarity index 89% rename from packages/cli/test/unit/sso/saml/samlHelpers.test.ts rename to packages/cli/src/sso/saml/__tests__/samlHelpers.test.ts index f6c35ff67e..17355524d3 100644 --- a/packages/cli/test/unit/sso/saml/samlHelpers.test.ts +++ b/packages/cli/src/sso/saml/__tests__/samlHelpers.test.ts @@ -1,8 +1,8 @@ import { User } from '@/databases/entities/User'; import { generateNanoId } from '@/databases/utils/generators'; -import * as helpers from '@/sso/saml/samlHelpers'; -import type { SamlUserAttributes } from '@/sso/saml/types/samlUserAttributes'; -import { mockInstance } from '../../../shared/mocking'; +import * as helpers from '@/sso/saml/saml-helpers'; +import type { SamlUserAttributes } from '@/sso/saml/types/saml-user-attributes'; +import { mockInstance } from '@test/mocking'; import { UserRepository } from '@/databases/repositories/user.repository'; import type { AuthIdentity } from '@/databases/entities/AuthIdentity'; import { AuthIdentityRepository } from '@/databases/repositories/authIdentity.repository'; diff --git a/packages/cli/src/sso/saml/middleware/samlEnabledMiddleware.ts b/packages/cli/src/sso/saml/middleware/saml-enabled-middleware.ts similarity index 97% rename from packages/cli/src/sso/saml/middleware/samlEnabledMiddleware.ts rename to packages/cli/src/sso/saml/middleware/saml-enabled-middleware.ts index e386541de0..e0bf2b1b01 100644 --- a/packages/cli/src/sso/saml/middleware/samlEnabledMiddleware.ts +++ b/packages/cli/src/sso/saml/middleware/saml-enabled-middleware.ts @@ -1,5 +1,5 @@ import type { RequestHandler } from 'express'; -import { isSamlLicensed, isSamlLicensedAndEnabled } from '../samlHelpers'; +import { isSamlLicensed, isSamlLicensedAndEnabled } from '../saml-helpers'; export const samlLicensedAndEnabledMiddleware: RequestHandler = (_, res, next) => { if (isSamlLicensedAndEnabled()) { diff --git a/packages/cli/src/sso/saml/routes/saml.controller.ee.ts b/packages/cli/src/sso/saml/routes/saml.controller.ee.ts index 344bd34e92..621d03bec1 100644 --- a/packages/cli/src/sso/saml/routes/saml.controller.ee.ts +++ b/packages/cli/src/sso/saml/routes/saml.controller.ee.ts @@ -15,19 +15,19 @@ import { getServiceProviderConfigTestReturnUrl, getServiceProviderEntityId, getServiceProviderReturnUrl, -} from '../serviceProvider.ee'; -import { getSamlConnectionTestSuccessView } from '../views/samlConnectionTestSuccess'; -import { getSamlConnectionTestFailedView } from '../views/samlConnectionTestFailed'; -import { isConnectionTestRequest, isSamlLicensedAndEnabled } from '../samlHelpers'; +} from '../service-provider.ee'; +import { getSamlConnectionTestSuccessView } from '../views/saml-connection-test-success'; +import { getSamlConnectionTestFailedView } from '../views/saml-connection-test-failed'; +import { isConnectionTestRequest, isSamlLicensedAndEnabled } from '../saml-helpers'; import type { SamlLoginBinding } from '../types'; import { samlLicensedAndEnabledMiddleware, samlLicensedMiddleware, -} from '../middleware/samlEnabledMiddleware'; +} from '../middleware/saml-enabled-middleware'; import { SamlService } from '../saml.service.ee'; import { SamlConfiguration } from '../types/requests'; -import { getInitSSOFormView } from '../views/initSsoPost'; -import { EventService } from '@/eventbus/event.service'; +import { getInitSSOFormView } from '../views/init-sso-post'; +import { EventService } from '@/events/event.service'; @RestController('/sso/saml') export class SamlController { diff --git a/packages/cli/src/sso/saml/samlHelpers.ts b/packages/cli/src/sso/saml/saml-helpers.ts similarity index 95% rename from packages/cli/src/sso/saml/samlHelpers.ts rename to packages/cli/src/sso/saml/saml-helpers.ts index d7f53900e5..c2dec7637f 100644 --- a/packages/cli/src/sso/saml/samlHelpers.ts +++ b/packages/cli/src/sso/saml/saml-helpers.ts @@ -9,20 +9,20 @@ import { UserRepository } from '@db/repositories/user.repository'; import { AuthIdentityRepository } from '@db/repositories/authIdentity.repository'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; import { AuthError } from '@/errors/response-errors/auth.error'; -import { License } from '@/License'; +import { License } from '@/license'; import { PasswordUtility } from '@/services/password.utility'; -import type { SamlPreferences } from './types/samlPreferences'; -import type { SamlUserAttributes } from './types/samlUserAttributes'; -import type { SamlAttributeMapping } from './types/samlAttributeMapping'; +import type { SamlPreferences } from './types/saml-preferences'; +import type { SamlUserAttributes } from './types/saml-user-attributes'; +import type { SamlAttributeMapping } from './types/saml-attribute-mapping'; import { SAML_LOGIN_ENABLED, SAML_LOGIN_LABEL } from './constants'; import { getCurrentAuthenticationMethod, isEmailCurrentAuthenticationMethod, isSamlCurrentAuthenticationMethod, setCurrentAuthenticationMethod, -} from '../ssoHelpers'; -import { getServiceProviderConfigTestReturnUrl } from './serviceProvider.ee'; +} from '../sso-helpers'; +import { getServiceProviderConfigTestReturnUrl } from './service-provider.ee'; import type { SamlConfiguration } from './types/requests'; /** diff --git a/packages/cli/src/sso/saml/samlValidator.ts b/packages/cli/src/sso/saml/saml-validator.ts similarity index 99% rename from packages/cli/src/sso/saml/samlValidator.ts rename to packages/cli/src/sso/saml/saml-validator.ts index 66be4c98e0..8e1e9dc328 100644 --- a/packages/cli/src/sso/saml/samlValidator.ts +++ b/packages/cli/src/sso/saml/saml-validator.ts @@ -1,6 +1,6 @@ import { Container } from 'typedi'; import type { XMLFileInfo } from 'xmllint-wasm'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; let xml: XMLFileInfo; let xmldsigCore: XMLFileInfo; diff --git a/packages/cli/src/sso/saml/saml.service.ee.ts b/packages/cli/src/sso/saml/saml.service.ee.ts index 1103bf73d9..a9411535cc 100644 --- a/packages/cli/src/sso/saml/saml.service.ee.ts +++ b/packages/cli/src/sso/saml/saml.service.ee.ts @@ -2,10 +2,10 @@ import type express from 'express'; import Container, { Service } from 'typedi'; import type { User } from '@db/entities/User'; import { ApplicationError, jsonParse } from 'n8n-workflow'; -import { getServiceProviderInstance } from './serviceProvider.ee'; -import type { SamlUserAttributes } from './types/samlUserAttributes'; -import { isSsoJustInTimeProvisioningEnabled } from '../ssoHelpers'; -import type { SamlPreferences } from './types/samlPreferences'; +import { getServiceProviderInstance } from './service-provider.ee'; +import type { SamlUserAttributes } from './types/saml-user-attributes'; +import { isSsoJustInTimeProvisioningEnabled } from '../sso-helpers'; +import type { SamlPreferences } from './types/saml-preferences'; import { SAML_PREFERENCES_DB_KEY } from './constants'; import type { IdentityProviderInstance, ServiceProviderInstance } from 'samlify'; import type { BindingContext, PostBindingContext } from 'samlify/types/src/entity'; @@ -18,13 +18,13 @@ import { setSamlLoginEnabled, setSamlLoginLabel, updateUserFromSamlAttributes, -} from './samlHelpers'; +} from './saml-helpers'; import type { Settings } from '@db/entities/Settings'; import axios from 'axios'; import https from 'https'; import type { SamlLoginBinding } from './types'; -import { validateMetadata, validateResponse } from './samlValidator'; -import { Logger } from '@/Logger'; +import { validateMetadata, validateResponse } from './saml-validator'; +import { Logger } from '@/logger'; import { UserRepository } from '@db/repositories/user.repository'; import { SettingsRepository } from '@db/repositories/settings.repository'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; diff --git a/packages/cli/src/sso/saml/serviceProvider.ee.ts b/packages/cli/src/sso/saml/service-provider.ee.ts similarity index 96% rename from packages/cli/src/sso/saml/serviceProvider.ee.ts rename to packages/cli/src/sso/saml/service-provider.ee.ts index 088457ca9c..4630f7c01a 100644 --- a/packages/cli/src/sso/saml/serviceProvider.ee.ts +++ b/packages/cli/src/sso/saml/service-provider.ee.ts @@ -1,7 +1,7 @@ import { Container } from 'typedi'; import type { ServiceProviderInstance } from 'samlify'; import { UrlService } from '@/services/url.service'; -import type { SamlPreferences } from './types/samlPreferences'; +import type { SamlPreferences } from './types/saml-preferences'; let serviceProviderInstance: ServiceProviderInstance | undefined; diff --git a/packages/cli/src/sso/saml/types/requests.ts b/packages/cli/src/sso/saml/types/requests.ts index 66bdc553d3..c333c8c361 100644 --- a/packages/cli/src/sso/saml/types/requests.ts +++ b/packages/cli/src/sso/saml/types/requests.ts @@ -1,5 +1,5 @@ import type { AuthenticatedRequest, AuthlessRequest } from '@/requests'; -import type { SamlPreferences } from './samlPreferences'; +import type { SamlPreferences } from './saml-preferences'; export declare namespace SamlConfiguration { type Update = AuthenticatedRequest<{}, {}, SamlPreferences, {}>; diff --git a/packages/cli/src/sso/saml/types/samlAttributeMapping.ts b/packages/cli/src/sso/saml/types/saml-attribute-mapping.ts similarity index 100% rename from packages/cli/src/sso/saml/types/samlAttributeMapping.ts rename to packages/cli/src/sso/saml/types/saml-attribute-mapping.ts diff --git a/packages/cli/src/sso/saml/types/samlPreferences.ts b/packages/cli/src/sso/saml/types/saml-preferences.ts similarity index 94% rename from packages/cli/src/sso/saml/types/samlPreferences.ts rename to packages/cli/src/sso/saml/types/saml-preferences.ts index da02f1ebc4..6f4fc696ca 100644 --- a/packages/cli/src/sso/saml/types/samlPreferences.ts +++ b/packages/cli/src/sso/saml/types/saml-preferences.ts @@ -1,7 +1,7 @@ import { IsBoolean, IsObject, IsOptional, IsString } from 'class-validator'; import { SignatureConfig } from 'samlify/types/src/types'; import { SamlLoginBinding } from '.'; -import { SamlAttributeMapping } from './samlAttributeMapping'; +import { SamlAttributeMapping } from './saml-attribute-mapping'; export class SamlPreferences { @IsObject() diff --git a/packages/cli/src/sso/saml/types/samlUserAttributes.ts b/packages/cli/src/sso/saml/types/saml-user-attributes.ts similarity index 100% rename from packages/cli/src/sso/saml/types/samlUserAttributes.ts rename to packages/cli/src/sso/saml/types/saml-user-attributes.ts diff --git a/packages/cli/src/sso/saml/views/initSsoPost.ts b/packages/cli/src/sso/saml/views/init-sso-post.ts similarity index 100% rename from packages/cli/src/sso/saml/views/initSsoPost.ts rename to packages/cli/src/sso/saml/views/init-sso-post.ts diff --git a/packages/cli/src/sso/saml/views/samlConnectionTestFailed.ts b/packages/cli/src/sso/saml/views/saml-connection-test-failed.ts similarity index 95% rename from packages/cli/src/sso/saml/views/samlConnectionTestFailed.ts rename to packages/cli/src/sso/saml/views/saml-connection-test-failed.ts index 8d9a3578f0..4ce2a3e3ac 100644 --- a/packages/cli/src/sso/saml/views/samlConnectionTestFailed.ts +++ b/packages/cli/src/sso/saml/views/saml-connection-test-failed.ts @@ -1,4 +1,4 @@ -import type { SamlUserAttributes } from '../types/samlUserAttributes'; +import type { SamlUserAttributes } from '../types/saml-user-attributes'; export function getSamlConnectionTestFailedView( message: string, diff --git a/packages/cli/src/sso/saml/views/samlConnectionTestSuccess.ts b/packages/cli/src/sso/saml/views/saml-connection-test-success.ts similarity index 95% rename from packages/cli/src/sso/saml/views/samlConnectionTestSuccess.ts rename to packages/cli/src/sso/saml/views/saml-connection-test-success.ts index 59e6aed263..f647527cd0 100644 --- a/packages/cli/src/sso/saml/views/samlConnectionTestSuccess.ts +++ b/packages/cli/src/sso/saml/views/saml-connection-test-success.ts @@ -1,4 +1,4 @@ -import type { SamlUserAttributes } from '../types/samlUserAttributes'; +import type { SamlUserAttributes } from '../types/saml-user-attributes'; export function getSamlConnectionTestSuccessView(attributes: SamlUserAttributes): string { return ` diff --git a/packages/cli/src/sso/ssoHelpers.ts b/packages/cli/src/sso/sso-helpers.ts similarity index 100% rename from packages/cli/src/sso/ssoHelpers.ts rename to packages/cli/src/sso/sso-helpers.ts diff --git a/packages/cli/src/subworkflows/__tests__/subworkflow-policy-checker.test.ts b/packages/cli/src/subworkflows/__tests__/subworkflow-policy-checker.test.ts index c20d8fdb2c..d4d58051a0 100644 --- a/packages/cli/src/subworkflows/__tests__/subworkflow-policy-checker.test.ts +++ b/packages/cli/src/subworkflows/__tests__/subworkflow-policy-checker.test.ts @@ -9,7 +9,7 @@ import { mock } from 'jest-mock-extended'; import { SubworkflowPolicyChecker } from '../subworkflow-policy-checker.service'; import type { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; -import type { License } from '@/License'; +import type { License } from '@/license'; import type { GlobalConfig } from '@n8n/config'; const toTargetCallErrorMsg = (subworkflowId: string) => diff --git a/packages/cli/src/subworkflows/subworkflow-policy-checker.service.ts b/packages/cli/src/subworkflows/subworkflow-policy-checker.service.ts index d2448cabaf..09f46eadf0 100644 --- a/packages/cli/src/subworkflows/subworkflow-policy-checker.service.ts +++ b/packages/cli/src/subworkflows/subworkflow-policy-checker.service.ts @@ -1,10 +1,13 @@ import { Service } from 'typedi'; -import { WorkflowOperationError } from 'n8n-workflow'; import { GlobalConfig } from '@n8n/config'; -import { Logger } from '@/Logger'; -import { License } from '@/License'; +import { Logger } from '@/logger'; +import { License } from '@/license'; import { OwnershipService } from '@/services/ownership.service'; -import type { Workflow, INode } from 'n8n-workflow'; +import type { Workflow, INode, WorkflowSettings } from 'n8n-workflow'; +import { SubworkflowPolicyDenialError } from '@/errors/subworkflow-policy-denial.error'; + +type Policy = WorkflowSettings.CallerPolicy; +type DenialPolicy = Exclude; @Service() export class SubworkflowPolicyChecker { @@ -15,97 +18,103 @@ export class SubworkflowPolicyChecker { private readonly globalConfig: GlobalConfig, ) {} + /** + * Check whether the parent workflow is allowed to call the subworkflow. + */ async check(subworkflow: Workflow, parentWorkflowId: string, node?: INode) { - /** - * Important considerations: both the current workflow and the parent can have empty IDs. - * This happens when a user is executing an unsaved workflow manually running a workflow - * loaded from a file or code, for instance. - * This is an important topic to keep in mind for all security checks - */ - if (!subworkflow.id) { - // It's a workflow from code and not loaded from DB - // No checks are necessary since it doesn't have any sort of settings - return; - } + const { id: subworkflowId } = subworkflow; - let policy = - subworkflow.settings?.callerPolicy ?? this.globalConfig.workflows.callerPolicyDefaultOption; + if (!subworkflowId) return; // e.g. when running a subworkflow loaded from a file - const isSharingEnabled = this.license.isSharingEnabled(); + const policy = this.findPolicy(subworkflow); - if (!isSharingEnabled) { - // Community version allows only same owner workflows - policy = 'workflowsFromSameOwner'; - } + if (policy === 'any') return; - const parentWorkflowOwner = - await this.ownershipService.getWorkflowProjectCached(parentWorkflowId); + const { parentWorkflowProject, subworkflowProject } = await this.findProjects({ + parentWorkflowId, + subworkflowId, + }); - const subworkflowOwner = await this.ownershipService.getWorkflowProjectCached(subworkflow.id); + const areOwnedBySameProject = parentWorkflowProject.id === subworkflowProject.id; - const description = - subworkflowOwner.id === parentWorkflowOwner.id - ? 'Change the settings of the sub-workflow so it can be called by this one.' - : `An admin for the ${subworkflowOwner.name} project can make this change. You may need to tell them the ID of the sub-workflow, which is ${subworkflow.id}`; + if ( + policy === 'none' || + (policy === 'workflowsFromAList' && !this.hasParentListed(subworkflow, parentWorkflowId)) || + (policy === 'workflowsFromSameOwner' && !areOwnedBySameProject) + ) { + this.logDenial({ parentWorkflowId, subworkflowId, policy }); - const errorToThrow = new WorkflowOperationError( - `Target workflow ID ${subworkflow.id} may not be called`, - node, - description, - ); - - if (policy === 'none') { - this.logger.warn('[PermissionChecker] Subworkflow execution denied', { - callerWorkflowId: parentWorkflowId, - subworkflowId: subworkflow.id, - reason: 'Subworkflow may not be called', - policy, - isSharingEnabled, + throw new SubworkflowPolicyDenialError({ + subworkflowId, + subworkflowProject, + areOwnedBySameProject, + node, }); - throw errorToThrow; - } - - if (policy === 'workflowsFromAList') { - if (parentWorkflowId === undefined) { - this.logger.warn('[PermissionChecker] Subworkflow execution denied', { - reason: 'Subworkflow may be called only by workflows from an allowlist', - callerWorkflowId: parentWorkflowId, - subworkflowId: subworkflow.id, - policy, - isSharingEnabled, - }); - throw errorToThrow; - } - - const allowedCallerIds = subworkflow.settings.callerIds - ?.split(',') - .map((id) => id.trim()) - .filter((id) => id !== ''); - - if (!allowedCallerIds?.includes(parentWorkflowId)) { - this.logger.warn('[PermissionChecker] Subworkflow execution denied', { - reason: 'Subworkflow may be called only by workflows from an allowlist', - callerWorkflowId: parentWorkflowId, - subworkflowId: subworkflow.id, - allowlist: allowedCallerIds, - policy, - isSharingEnabled, - }); - throw errorToThrow; - } - } - - if (policy === 'workflowsFromSameOwner' && subworkflowOwner?.id !== parentWorkflowOwner.id) { - this.logger.warn('[PermissionChecker] Subworkflow execution denied', { - reason: 'Subworkflow may be called only by workflows owned by the same project', - callerWorkflowId: parentWorkflowId, - subworkflowId: subworkflow.id, - callerProjectId: parentWorkflowOwner.id, - subworkflowProjectId: subworkflowOwner.id, - policy, - isSharingEnabled, - }); - throw errorToThrow; } } + + /** + * Find the subworkflow's caller policy. + */ + private findPolicy(subworkflow: Workflow): WorkflowSettings.CallerPolicy { + if (!this.license.isSharingEnabled()) return 'workflowsFromSameOwner'; + + return ( + subworkflow.settings?.callerPolicy ?? this.globalConfig.workflows.callerPolicyDefaultOption + ); + } + + /** + * Find the projects that own the parent workflow and the subworkflow. + */ + private async findProjects({ + parentWorkflowId, + subworkflowId, + }: { + parentWorkflowId: string; + subworkflowId: string; + }) { + const [parentWorkflowProject, subworkflowProject] = await Promise.all([ + this.ownershipService.getWorkflowProjectCached(parentWorkflowId), + this.ownershipService.getWorkflowProjectCached(subworkflowId), + ]); + + return { parentWorkflowProject, subworkflowProject }; + } + + /** + * Whether the subworkflow has the parent workflow listed as a caller. + */ + private hasParentListed(subworkflow: Workflow, parentWorkflowId: string) { + const callerIds = + subworkflow.settings.callerIds + ?.split(',') + .map((id) => id.trim()) + .filter((id) => id !== '') ?? []; + + return callerIds.includes(parentWorkflowId); + } + + private readonly denialReasons: Record = { + none: 'Subworkflow may not be called by any workflow', + workflowsFromAList: 'Subworkflow may be called only by workflows from an allowlist', + workflowsFromSameOwner: 'Subworkflow may be called only by workflows owned by the same project', + }; + + private logDenial({ + parentWorkflowId, + subworkflowId, + policy, + }: { + parentWorkflowId: string; + subworkflowId: string; + policy: DenialPolicy; + }) { + this.logger.warn('[SubworkflowPolicyChecker] Subworkflow execution denied', { + reason: this.denialReasons[policy], + parentWorkflowId, + subworkflowId, + isSharingEnabled: this.license.isSharingEnabled(), + }); + } } diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/src/telemetry/__tests__/telemetry.test.ts similarity index 59% rename from packages/cli/test/unit/Telemetry.test.ts rename to packages/cli/src/telemetry/__tests__/telemetry.test.ts index 25f4a80867..ca30c86ec4 100644 --- a/packages/cli/test/unit/Telemetry.test.ts +++ b/packages/cli/src/telemetry/__tests__/telemetry.test.ts @@ -1,11 +1,10 @@ import type RudderStack from '@rudderstack/rudder-sdk-node'; import { Telemetry } from '@/telemetry'; import config from '@/config'; -import { flushPromises } from './Helpers'; import { PostHogClient } from '@/posthog'; import { mock } from 'jest-mock-extended'; import { InstanceSettings } from 'n8n-core'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance } from '@test/mocking'; jest.unmock('@/telemetry'); jest.mock('@/posthog'); @@ -14,11 +13,7 @@ describe('Telemetry', () => { let startPulseSpy: jest.SpyInstance; const spyTrack = jest.spyOn(Telemetry.prototype, 'track').mockName('track'); - const mockRudderStack: Pick = { - flush: (resolve) => resolve?.(), - identify: (data, resolve) => resolve?.(), - track: (data, resolve) => resolve?.(), - }; + const mockRudderStack = mock(); let telemetry: Telemetry; const instanceId = 'Telemetry unit test'; @@ -26,9 +21,9 @@ describe('Telemetry', () => { const instanceSettings = mockInstance(InstanceSettings, { instanceId }); beforeAll(() => { - startPulseSpy = jest - .spyOn(Telemetry.prototype as any, 'startPulse') - .mockImplementation(() => {}); + // @ts-expect-error Spying on private method + startPulseSpy = jest.spyOn(Telemetry.prototype, 'startPulse').mockImplementation(() => {}); + jest.useFakeTimers(); jest.setSystemTime(testDateTime); config.set('diagnostics.enabled', true); @@ -39,7 +34,7 @@ describe('Telemetry', () => { jest.clearAllTimers(); jest.useRealTimers(); startPulseSpy.mockRestore(); - await telemetry.trackN8nStop(); + await telemetry.stopTracking(); }); beforeEach(async () => { @@ -49,18 +44,12 @@ describe('Telemetry', () => { await postHog.init(); telemetry = new Telemetry(mock(), postHog, mock(), instanceSettings, mock()); - (telemetry as any).rudderStack = mockRudderStack; + // @ts-expect-error Assigning to private property + telemetry.rudderStack = mockRudderStack; }); afterEach(async () => { - await telemetry.trackN8nStop(); - }); - - describe('trackN8nStop', () => { - test('should call track method', async () => { - await telemetry.trackN8nStop(); - expect(spyTrack).toHaveBeenCalledTimes(1); - }); + await telemetry.stopTracking(); }); describe('trackWorkflowExecution', () => { @@ -79,30 +68,30 @@ describe('Telemetry', () => { payload.is_manual = true; payload.success = true; const execTime1 = fakeJestSystemTime('2022-01-01 12:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); payload.is_manual = false; payload.success = true; const execTime2 = fakeJestSystemTime('2022-01-01 13:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); payload.is_manual = true; payload.success = false; const execTime3 = fakeJestSystemTime('2022-01-01 14:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); payload.is_manual = false; payload.success = false; const execTime4 = fakeJestSystemTime('2022-01-01 15:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(0); @@ -127,9 +116,9 @@ describe('Telemetry', () => { }; const execTime1 = fakeJestSystemTime('2022-01-01 12:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); let execBuffer = telemetry.getCountsBuffer(); @@ -140,9 +129,9 @@ describe('Telemetry', () => { payload.error_node_type = 'n8n-nodes-base.node-type'; fakeJestSystemTime('2022-01-01 13:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); fakeJestSystemTime('2022-01-01 12:30:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); execBuffer = telemetry.getCountsBuffer(); @@ -163,7 +152,7 @@ describe('Telemetry', () => { // successful execution const execTime1 = fakeJestSystemTime('2022-01-01 12:00:00'); - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(0); @@ -179,7 +168,7 @@ describe('Telemetry', () => { payload.error_node_type = 'n8n-nodes-base.merge'; payload.workflow_id = '2'; - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(0); @@ -198,12 +187,12 @@ describe('Telemetry', () => { payload.error_node_type = 'n8n-nodes-base.merge'; payload.workflow_id = '2'; - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); payload.error_node_type = 'n8n-nodes-base.merge'; payload.workflow_id = '1'; - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(0); execBuffer = telemetry.getCountsBuffer(); @@ -225,7 +214,7 @@ describe('Telemetry', () => { const execTime2 = fakeJestSystemTime('2022-01-01 12:00:00'); payload.error_node_type = 'custom-package.custom-node'; payload.success = false; - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(0); @@ -249,7 +238,7 @@ describe('Telemetry', () => { payload.success = false; payload.error_node_type = 'n8n-nodes-base.merge'; payload.is_manual = true; - await telemetry.trackWorkflowExecution(payload); + telemetry.trackWorkflowExecution(payload); expect(spyTrack).toHaveBeenCalledTimes(1); @@ -269,162 +258,6 @@ describe('Telemetry', () => { expect(execBuffer['2'].prod_success?.first).toEqual(execTime1); }); }); - - describe('pulse', () => { - let pulseSpy: jest.SpyInstance; - beforeAll(() => { - startPulseSpy.mockRestore(); - }); - - beforeEach(() => { - fakeJestSystemTime(testDateTime); - pulseSpy = jest.spyOn(Telemetry.prototype as any, 'pulse').mockName('pulseSpy'); - }); - - afterEach(() => { - pulseSpy.mockClear(); - }); - - xtest('should trigger pulse in intervals', async () => { - expect(pulseSpy).toBeCalledTimes(0); - - jest.advanceTimersToNextTimer(); - await flushPromises(); - - expect(pulseSpy).toBeCalledTimes(1); - expect(spyTrack).toHaveBeenCalledTimes(1); - expect(spyTrack).toHaveBeenCalledWith('pulse', { - plan_name_current: 'Community', - quota: -1, - usage: 0, - }); - - jest.advanceTimersToNextTimer(); - - await flushPromises(); - - expect(pulseSpy).toBeCalledTimes(2); - expect(spyTrack).toHaveBeenCalledTimes(2); - expect(spyTrack).toHaveBeenCalledWith('pulse', { - plan_name_current: 'Community', - quota: -1, - usage: 0, - }); - }); - - xtest('should track workflow counts correctly', async () => { - expect(pulseSpy).toBeCalledTimes(0); - - let execBuffer = telemetry.getCountsBuffer(); - - // expect clear counters on start - expect(Object.keys(execBuffer).length).toBe(0); - - const payload = { - workflow_id: '1', - is_manual: true, - success: true, - error_node_type: 'custom-nodes-base.node-type', - }; - - await telemetry.trackWorkflowExecution(payload); - await telemetry.trackWorkflowExecution(payload); - - payload.is_manual = false; - payload.success = true; - await telemetry.trackWorkflowExecution(payload); - await telemetry.trackWorkflowExecution(payload); - - payload.is_manual = true; - payload.success = false; - await telemetry.trackWorkflowExecution(payload); - await telemetry.trackWorkflowExecution(payload); - - payload.is_manual = false; - payload.success = false; - await telemetry.trackWorkflowExecution(payload); - await telemetry.trackWorkflowExecution(payload); - - payload.workflow_id = '2'; - await telemetry.trackWorkflowExecution(payload); - await telemetry.trackWorkflowExecution(payload); - - expect(spyTrack).toHaveBeenCalledTimes(0); - expect(pulseSpy).toBeCalledTimes(0); - - jest.advanceTimersToNextTimer(); - - execBuffer = telemetry.getCountsBuffer(); - - await flushPromises(); - - expect(pulseSpy).toBeCalledTimes(1); - expect(spyTrack).toHaveBeenCalledTimes(3); - expect(spyTrack).toHaveBeenNthCalledWith( - 1, - 'Workflow execution count', - { - event_version: '2', - workflow_id: '1', - user_id: undefined, - manual_error: { - count: 2, - first: testDateTime, - }, - manual_success: { - count: 2, - first: testDateTime, - }, - prod_error: { - count: 2, - first: testDateTime, - }, - prod_success: { - count: 2, - first: testDateTime, - }, - }, - { withPostHog: true }, - ); - expect(spyTrack).toHaveBeenNthCalledWith( - 2, - 'Workflow execution count', - { - event_version: '2', - workflow_id: '2', - user_id: undefined, - prod_error: { - count: 2, - first: testDateTime, - }, - }, - { withPostHog: true }, - ); - expect(spyTrack).toHaveBeenNthCalledWith(3, 'pulse', { - plan_name_current: 'Community', - quota: -1, - usage: 0, - }); - expect(Object.keys(execBuffer).length).toBe(0); - - // Adding a second step here because we believe PostHog may use timers for sending data - // and adding posthog to the above metric was causing the pulseSpy timer to not be ran - jest.advanceTimersToNextTimer(); - - execBuffer = telemetry.getCountsBuffer(); - expect(Object.keys(execBuffer).length).toBe(0); - - // @TODO: Flushing promises here is not working - - // expect(pulseSpy).toBeCalledTimes(2); - // expect(spyTrack).toHaveBeenCalledTimes(4); - // expect(spyTrack).toHaveBeenNthCalledWith(4, 'pulse', { - // plan_name_current: 'Community', - // quota: -1, - // usage: 0, - // }); - }); - }); }); const fakeJestSystemTime = (dateTime: string | Date): Date => { diff --git a/packages/cli/src/telemetry/index.ts b/packages/cli/src/telemetry/index.ts index f63caf9b2c..2aec0f07d2 100644 --- a/packages/cli/src/telemetry/index.ts +++ b/packages/cli/src/telemetry/index.ts @@ -7,14 +7,15 @@ import { InstanceSettings } from 'n8n-core'; import config from '@/config'; import type { IExecutionTrackProperties } from '@/Interfaces'; -import { Logger } from '@/Logger'; -import { License } from '@/License'; -import { N8N_VERSION } from '@/constants'; +import { Logger } from '@/logger'; +import { License } from '@/license'; +import { LOWEST_SHUTDOWN_PRIORITY, N8N_VERSION } from '@/constants'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { SourceControlPreferencesService } from '../environments/sourceControl/sourceControlPreferences.service.ee'; import { UserRepository } from '@db/repositories/user.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; +import { OnShutdown } from '@/decorators/on-shutdown'; type ExecutionTrackDataKey = 'manual_error' | 'manual_success' | 'prod_error' | 'prod_success'; @@ -88,30 +89,28 @@ export class Telemetry { ); // every 6 hours } - private async pulse(): Promise { + private async pulse() { if (!this.rudderStack) { return; } - const allPromises = Object.keys(this.executionCountsBuffer) - .filter((workflowId) => { - const data = this.executionCountsBuffer[workflowId]; - const sum = - (data.manual_error?.count ?? 0) + - (data.manual_success?.count ?? 0) + - (data.prod_error?.count ?? 0) + - (data.prod_success?.count ?? 0); - return sum > 0; - }) - .map(async (workflowId) => { - const promise = this.track('Workflow execution count', { - event_version: '2', - workflow_id: workflowId, - ...this.executionCountsBuffer[workflowId], - }); + const workflowIdsToReport = Object.keys(this.executionCountsBuffer).filter((workflowId) => { + const data = this.executionCountsBuffer[workflowId]; + const sum = + (data.manual_error?.count ?? 0) + + (data.manual_success?.count ?? 0) + + (data.prod_error?.count ?? 0) + + (data.prod_success?.count ?? 0); + return sum > 0; + }); - return await promise; + for (const workflowId of workflowIdsToReport) { + this.track('Workflow execution count', { + event_version: '2', + workflow_id: workflowId, + ...this.executionCountsBuffer[workflowId], }); + } this.executionCountsBuffer = {}; @@ -131,11 +130,11 @@ export class Telemetry { team_projects: (await Container.get(ProjectRepository).getProjectCounts()).team, project_role_count: await Container.get(ProjectRelationRepository).countUsersByRole(), }; - allPromises.push(this.track('pulse', pulsePacket)); - return await Promise.all(allPromises); + + this.track('pulse', pulsePacket); } - async trackWorkflowExecution(properties: IExecutionTrackProperties): Promise { + trackWorkflowExecution(properties: IExecutionTrackProperties) { if (this.rudderStack) { const execTime = new Date(); const workflowId = properties.workflow_id; @@ -164,66 +163,59 @@ export class Telemetry { properties.is_manual && properties.error_node_type?.startsWith('n8n-nodes-base') ) { - void this.track('Workflow execution errored', properties); + this.track('Workflow execution errored', properties); } } } - async trackN8nStop(): Promise { + @OnShutdown(LOWEST_SHUTDOWN_PRIORITY) + async stopTracking(): Promise { clearInterval(this.pulseIntervalReference); - await this.track('User instance stopped'); - void Promise.all([this.postHog.stop(), this.rudderStack?.flush()]); + + await Promise.all([this.postHog.stop(), this.rudderStack?.flush()]); } - async identify(traits?: { - [key: string]: string | number | boolean | object | undefined | null; - }): Promise { + identify(traits?: { [key: string]: string | number | boolean | object | undefined | null }) { + if (!this.rudderStack) { + return; + } + const { instanceId } = this.instanceSettings; - return await new Promise((resolve) => { - if (this.rudderStack) { - this.rudderStack.identify( - { - userId: instanceId, - traits: { ...traits, instanceId }, - }, - resolve, - ); - } else { - resolve(); - } + + this.rudderStack.identify({ + userId: instanceId, + traits: { ...traits, instanceId }, }); } - async track( + track( eventName: string, properties: ITelemetryTrackProperties = {}, { withPostHog } = { withPostHog: false }, // whether to additionally track with PostHog - ): Promise { + ) { + if (!this.rudderStack) { + return; + } + const { instanceId } = this.instanceSettings; - return await new Promise((resolve) => { - if (this.rudderStack) { - const { user_id } = properties; - const updatedProperties = { - ...properties, - instance_id: instanceId, - version_cli: N8N_VERSION, - }; + const { user_id } = properties; + const updatedProperties = { + ...properties, + instance_id: instanceId, + version_cli: N8N_VERSION, + }; - const payload = { - userId: `${instanceId}${user_id ? `#${user_id}` : ''}`, - event: eventName, - properties: updatedProperties, - }; + const payload = { + userId: `${instanceId}${user_id ? `#${user_id}` : ''}`, + event: eventName, + properties: updatedProperties, + }; - if (withPostHog) { - this.postHog?.track(payload); - } + if (withPostHog) { + this.postHog?.track(payload); + } - return this.rudderStack.track(payload, resolve); - } - - return resolve(); - }); + return this.rudderStack.track(payload); } // test helpers diff --git a/packages/cli/src/telemetry/telemetry-event-relay.service.ts b/packages/cli/src/telemetry/telemetry-event-relay.service.ts deleted file mode 100644 index 915ea0dd84..0000000000 --- a/packages/cli/src/telemetry/telemetry-event-relay.service.ts +++ /dev/null @@ -1,360 +0,0 @@ -import { Service } from 'typedi'; -import { EventService } from '@/eventbus/event.service'; -import type { Event } from '@/eventbus/event.types'; -import { Telemetry } from '.'; -import config from '@/config'; - -@Service() -export class TelemetryEventRelay { - constructor( - private readonly eventService: EventService, - private readonly telemetry: Telemetry, - ) {} - - async init() { - if (!config.getEnv('diagnostics.enabled')) return; - - await this.telemetry.init(); - - this.setupHandlers(); - } - - private setupHandlers() { - this.eventService.on('team-project-updated', (event) => this.teamProjectUpdated(event)); - this.eventService.on('team-project-deleted', (event) => this.teamProjectDeleted(event)); - this.eventService.on('team-project-created', (event) => this.teamProjectCreated(event)); - this.eventService.on('source-control-settings-updated', (event) => - this.sourceControlSettingsUpdated(event), - ); - this.eventService.on('source-control-user-started-pull-ui', (event) => - this.sourceControlUserStartedPullUi(event), - ); - this.eventService.on('source-control-user-finished-pull-ui', (event) => - this.sourceControlUserFinishedPullUi(event), - ); - this.eventService.on('source-control-user-pulled-api', (event) => - this.sourceControlUserPulledApi(event), - ); - this.eventService.on('source-control-user-started-push-ui', (event) => - this.sourceControlUserStartedPushUi(event), - ); - this.eventService.on('source-control-user-finished-push-ui', (event) => - this.sourceControlUserFinishedPushUi(event), - ); - this.eventService.on('license-renewal-attempted', (event) => { - this.licenseRenewalAttempted(event); - }); - this.eventService.on('variable-created', () => this.variableCreated()); - this.eventService.on('external-secrets-provider-settings-saved', (event) => { - this.externalSecretsProviderSettingsSaved(event); - }); - this.eventService.on('public-api-invoked', (event) => { - this.publicApiInvoked(event); - }); - this.eventService.on('public-api-key-created', (event) => { - this.publicApiKeyCreated(event); - }); - this.eventService.on('public-api-key-deleted', (event) => { - this.publicApiKeyDeleted(event); - }); - this.eventService.on('community-package-installed', (event) => { - this.communityPackageInstalled(event); - }); - this.eventService.on('community-package-updated', (event) => { - this.communityPackageUpdated(event); - }); - this.eventService.on('community-package-deleted', (event) => { - this.communityPackageDeleted(event); - }); - this.eventService.on('ldap-general-sync-finished', (event) => { - this.ldapGeneralSyncFinished(event); - }); - this.eventService.on('ldap-settings-updated', (event) => { - this.ldapSettingsUpdated(event); - }); - this.eventService.on('ldap-login-sync-failed', (event) => { - this.ldapLoginSyncFailed(event); - }); - this.eventService.on('login-failed-due-to-ldap-disabled', (event) => { - this.loginFailedDueToLdapDisabled(event); - }); - } - - private teamProjectUpdated({ userId, role, members, projectId }: Event['team-project-updated']) { - void this.telemetry.track('Project settings updated', { - user_id: userId, - role, - // eslint-disable-next-line @typescript-eslint/no-shadow - members: members.map(({ userId: user_id, role }) => ({ user_id, role })), - project_id: projectId, - }); - } - - private teamProjectDeleted({ - userId, - role, - projectId, - removalType, - targetProjectId, - }: Event['team-project-deleted']) { - void this.telemetry.track('User deleted project', { - user_id: userId, - role, - project_id: projectId, - removal_type: removalType, - target_project_id: targetProjectId, - }); - } - - private teamProjectCreated({ userId, role }: Event['team-project-created']) { - void this.telemetry.track('User created project', { - user_id: userId, - role, - }); - } - - private sourceControlSettingsUpdated({ - branchName, - readOnlyInstance, - repoType, - connected, - }: Event['source-control-settings-updated']) { - void this.telemetry.track('User updated source control settings', { - branch_name: branchName, - read_only_instance: readOnlyInstance, - repo_type: repoType, - connected, - }); - } - - private sourceControlUserStartedPullUi({ - workflowUpdates, - workflowConflicts, - credConflicts, - }: Event['source-control-user-started-pull-ui']) { - void this.telemetry.track('User started pull via UI', { - workflow_updates: workflowUpdates, - workflow_conflicts: workflowConflicts, - cred_conflicts: credConflicts, - }); - } - - private sourceControlUserFinishedPullUi({ - workflowUpdates, - }: Event['source-control-user-finished-pull-ui']) { - void this.telemetry.track('User finished pull via UI', { - workflow_updates: workflowUpdates, - }); - } - - private sourceControlUserPulledApi({ - workflowUpdates, - forced, - }: Event['source-control-user-pulled-api']) { - console.log('source-control-user-pulled-api', { - workflow_updates: workflowUpdates, - forced, - }); - void this.telemetry.track('User pulled via API', { - workflow_updates: workflowUpdates, - forced, - }); - } - - private sourceControlUserStartedPushUi({ - workflowsEligible, - workflowsEligibleWithConflicts, - credsEligible, - credsEligibleWithConflicts, - variablesEligible, - }: Event['source-control-user-started-push-ui']) { - void this.telemetry.track('User started push via UI', { - workflows_eligible: workflowsEligible, - workflows_eligible_with_conflicts: workflowsEligibleWithConflicts, - creds_eligible: credsEligible, - creds_eligible_with_conflicts: credsEligibleWithConflicts, - variables_eligible: variablesEligible, - }); - } - - private sourceControlUserFinishedPushUi({ - workflowsEligible, - workflowsPushed, - credsPushed, - variablesPushed, - }: Event['source-control-user-finished-push-ui']) { - void this.telemetry.track('User finished push via UI', { - workflows_eligible: workflowsEligible, - workflows_pushed: workflowsPushed, - creds_pushed: credsPushed, - variables_pushed: variablesPushed, - }); - } - - private licenseRenewalAttempted({ success }: Event['license-renewal-attempted']) { - void this.telemetry.track('Instance attempted to refresh license', { - success, - }); - } - - private variableCreated() { - void this.telemetry.track('User created variable'); - } - - private externalSecretsProviderSettingsSaved({ - userId, - vaultType, - isValid, - isNew, - errorMessage, - }: Event['external-secrets-provider-settings-saved']) { - void this.telemetry.track('User updated external secrets settings', { - user_id: userId, - vault_type: vaultType, - is_valid: isValid, - is_new: isNew, - error_message: errorMessage, - }); - } - - private publicApiInvoked({ userId, path, method, apiVersion }: Event['public-api-invoked']) { - void this.telemetry.track('User invoked API', { - user_id: userId, - path, - method, - api_version: apiVersion, - }); - } - - private publicApiKeyCreated(event: Event['public-api-key-created']) { - const { user, publicApi } = event; - - void this.telemetry.track('API key created', { - user_id: user.id, - public_api: publicApi, - }); - } - - private publicApiKeyDeleted(event: Event['public-api-key-deleted']) { - const { user, publicApi } = event; - - void this.telemetry.track('API key deleted', { - user_id: user.id, - public_api: publicApi, - }); - } - - private communityPackageInstalled({ - user, - inputString, - packageName, - success, - packageVersion, - packageNodeNames, - packageAuthor, - packageAuthorEmail, - failureReason, - }: Event['community-package-installed']) { - void this.telemetry.track('cnr package install finished', { - user_id: user.id, - input_string: inputString, - package_name: packageName, - success, - package_version: packageVersion, - package_node_names: packageNodeNames, - package_author: packageAuthor, - package_author_email: packageAuthorEmail, - failure_reason: failureReason, - }); - } - - private communityPackageUpdated({ - user, - packageName, - packageVersionCurrent, - packageVersionNew, - packageNodeNames, - packageAuthor, - packageAuthorEmail, - }: Event['community-package-updated']) { - void this.telemetry.track('cnr package updated', { - user_id: user.id, - package_name: packageName, - package_version_current: packageVersionCurrent, - package_version_new: packageVersionNew, - package_node_names: packageNodeNames, - package_author: packageAuthor, - package_author_email: packageAuthorEmail, - }); - } - - private communityPackageDeleted({ - user, - packageName, - packageVersion, - packageNodeNames, - packageAuthor, - packageAuthorEmail, - }: Event['community-package-deleted']) { - void this.telemetry.track('cnr package deleted', { - user_id: user.id, - package_name: packageName, - package_version: packageVersion, - package_node_names: packageNodeNames, - package_author: packageAuthor, - package_author_email: packageAuthorEmail, - }); - } - - private ldapGeneralSyncFinished({ - type, - succeeded, - usersSynced, - error, - }: Event['ldap-general-sync-finished']) { - void this.telemetry.track('Ldap general sync finished', { - type, - succeeded, - users_synced: usersSynced, - error, - }); - } - - private ldapSettingsUpdated({ - userId, - loginIdAttribute, - firstNameAttribute, - lastNameAttribute, - emailAttribute, - ldapIdAttribute, - searchPageSize, - searchTimeout, - synchronizationEnabled, - synchronizationInterval, - loginLabel, - loginEnabled, - }: Event['ldap-settings-updated']) { - void this.telemetry.track('User updated Ldap settings', { - user_id: userId, - loginIdAttribute, - firstNameAttribute, - lastNameAttribute, - emailAttribute, - ldapIdAttribute, - searchPageSize, - searchTimeout, - synchronizationEnabled, - synchronizationInterval, - loginLabel, - loginEnabled, - }); - } - - private ldapLoginSyncFailed({ error }: Event['ldap-login-sync-failed']) { - void this.telemetry.track('Ldap login sync failed', { error }); - } - - private loginFailedDueToLdapDisabled({ userId }: Event['login-failed-due-to-ldap-disabled']) { - void this.telemetry.track('User login failed since ldap disabled', { user_ud: userId }); - } -} diff --git a/packages/cli/src/UserManagement/email/Interfaces.ts b/packages/cli/src/user-management/email/Interfaces.ts similarity index 100% rename from packages/cli/src/UserManagement/email/Interfaces.ts rename to packages/cli/src/user-management/email/Interfaces.ts diff --git a/packages/cli/test/unit/UserManagementMailer.test.ts b/packages/cli/src/user-management/email/__tests__/user-management-mailer.test.ts similarity index 91% rename from packages/cli/test/unit/UserManagementMailer.test.ts rename to packages/cli/src/user-management/email/__tests__/user-management-mailer.test.ts index d39b27435a..a52a240bc4 100644 --- a/packages/cli/test/unit/UserManagementMailer.test.ts +++ b/packages/cli/src/user-management/email/__tests__/user-management-mailer.test.ts @@ -1,9 +1,9 @@ import type { GlobalConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; -import type { InviteEmailData, PasswordResetData } from '@/UserManagement/email/Interfaces'; -import { NodeMailer } from '@/UserManagement/email/NodeMailer'; -import { UserManagementMailer } from '@/UserManagement/email/UserManagementMailer'; +import type { InviteEmailData, PasswordResetData } from '@/user-management/email/Interfaces'; +import { NodeMailer } from '@/user-management/email/node-mailer'; +import { UserManagementMailer } from '@/user-management/email/user-management-mailer'; import { mockInstance } from '@test/mocking'; describe('UserManagementMailer', () => { diff --git a/packages/cli/src/user-management/email/index.ts b/packages/cli/src/user-management/email/index.ts new file mode 100644 index 0000000000..4b2a714fb7 --- /dev/null +++ b/packages/cli/src/user-management/email/index.ts @@ -0,0 +1,3 @@ +import { UserManagementMailer } from './user-management-mailer'; + +export { UserManagementMailer }; diff --git a/packages/cli/src/UserManagement/email/NodeMailer.ts b/packages/cli/src/user-management/email/node-mailer.ts similarity index 98% rename from packages/cli/src/UserManagement/email/NodeMailer.ts rename to packages/cli/src/user-management/email/node-mailer.ts index 8d27e2181b..d8834f9aae 100644 --- a/packages/cli/src/UserManagement/email/NodeMailer.ts +++ b/packages/cli/src/user-management/email/node-mailer.ts @@ -6,7 +6,7 @@ import type SMTPConnection from 'nodemailer/lib/smtp-connection'; import { GlobalConfig } from '@n8n/config'; import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { MailData, SendEmailResult } from './Interfaces'; @Service() diff --git a/packages/cli/src/UserManagement/email/templates/credentialsShared.html b/packages/cli/src/user-management/email/templates/credentialsShared.html similarity index 100% rename from packages/cli/src/UserManagement/email/templates/credentialsShared.html rename to packages/cli/src/user-management/email/templates/credentialsShared.html diff --git a/packages/cli/src/UserManagement/email/templates/instanceSetup.html b/packages/cli/src/user-management/email/templates/instanceSetup.html similarity index 100% rename from packages/cli/src/UserManagement/email/templates/instanceSetup.html rename to packages/cli/src/user-management/email/templates/instanceSetup.html diff --git a/packages/cli/src/UserManagement/email/templates/invite.html b/packages/cli/src/user-management/email/templates/invite.html similarity index 100% rename from packages/cli/src/UserManagement/email/templates/invite.html rename to packages/cli/src/user-management/email/templates/invite.html diff --git a/packages/cli/src/UserManagement/email/templates/passwordReset.html b/packages/cli/src/user-management/email/templates/passwordReset.html similarity index 100% rename from packages/cli/src/UserManagement/email/templates/passwordReset.html rename to packages/cli/src/user-management/email/templates/passwordReset.html diff --git a/packages/cli/src/UserManagement/email/templates/workflowShared.html b/packages/cli/src/user-management/email/templates/workflowShared.html similarity index 100% rename from packages/cli/src/UserManagement/email/templates/workflowShared.html rename to packages/cli/src/user-management/email/templates/workflowShared.html diff --git a/packages/cli/src/UserManagement/email/UserManagementMailer.ts b/packages/cli/src/user-management/email/user-management-mailer.ts similarity index 88% rename from packages/cli/src/UserManagement/email/UserManagementMailer.ts rename to packages/cli/src/user-management/email/user-management-mailer.ts index 531ce24cd6..0c5e021d94 100644 --- a/packages/cli/src/UserManagement/email/UserManagementMailer.ts +++ b/packages/cli/src/user-management/email/user-management-mailer.ts @@ -8,15 +8,14 @@ import { GlobalConfig } from '@n8n/config'; import type { User } from '@db/entities/User'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import { UserRepository } from '@db/repositories/user.repository'; -import { InternalHooks } from '@/InternalHooks'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { UrlService } from '@/services/url.service'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; import { toError } from '@/utils'; import type { InviteEmailData, PasswordResetData, SendEmailResult } from './Interfaces'; -import { NodeMailer } from './NodeMailer'; -import { EventService } from '@/eventbus/event.service'; +import { NodeMailer } from './node-mailer'; +import { EventService } from '@/events/event.service'; type Template = HandlebarsTemplateDelegate; type TemplateName = 'invite' | 'passwordReset' | 'workflowShared' | 'credentialsShared'; @@ -112,22 +111,18 @@ export class UserManagementMailer { this.logger.info('Sent workflow shared email successfully', { sharerId: sharer.id }); - void Container.get(InternalHooks).onUserTransactionalEmail({ - user_id: sharer.id, - message_type: 'Workflow shared', - public_api: false, + Container.get(EventService).emit('user-transactional-email-sent', { + userId: sharer.id, + messageType: 'Workflow shared', + publicApi: false, }); return result; } catch (e) { - void Container.get(InternalHooks).onEmailFailed({ - user: sharer, - message_type: 'Workflow shared', - public_api: false, - }); Container.get(EventService).emit('email-failed', { user: sharer, messageType: 'Workflow shared', + publicApi: false, }); const error = toError(e); @@ -171,22 +166,18 @@ export class UserManagementMailer { this.logger.info('Sent credentials shared email successfully', { sharerId: sharer.id }); - void Container.get(InternalHooks).onUserTransactionalEmail({ - user_id: sharer.id, - message_type: 'Credentials shared', - public_api: false, + Container.get(EventService).emit('user-transactional-email-sent', { + userId: sharer.id, + messageType: 'Credentials shared', + publicApi: false, }); return result; } catch (e) { - void Container.get(InternalHooks).onEmailFailed({ - user: sharer, - message_type: 'Credentials shared', - public_api: false, - }); Container.get(EventService).emit('email-failed', { user: sharer, messageType: 'Credentials shared', + publicApi: false, }); const error = toError(e); diff --git a/packages/cli/src/UserManagement/PermissionChecker.ts b/packages/cli/src/user-management/permission-checker.ts similarity index 100% rename from packages/cli/src/UserManagement/PermissionChecker.ts rename to packages/cli/src/user-management/permission-checker.ts diff --git a/packages/cli/src/validators/__tests__/no-url.validator.test.ts b/packages/cli/src/validators/__tests__/no-url.validator.test.ts new file mode 100644 index 0000000000..dafba595ef --- /dev/null +++ b/packages/cli/src/validators/__tests__/no-url.validator.test.ts @@ -0,0 +1,26 @@ +import { NoUrl } from '../no-url.validator'; +import { validate } from 'class-validator'; + +describe('NoUrl', () => { + class Entity { + @NoUrl() + name = ''; + } + + const entity = new Entity(); + + describe('URLs', () => { + const URLS = ['http://google.com', 'www.domain.tld', 'n8n.io']; + + for (const str of URLS) { + test(`should block ${str}`, async () => { + entity.name = str; + const errors = await validate(entity); + expect(errors).toHaveLength(1); + const [error] = errors; + expect(error.property).toEqual('name'); + expect(error.constraints).toEqual({ NoUrl: 'Potentially malicious string' }); + }); + } + }); +}); diff --git a/packages/cli/src/validators/__tests__/no-xss.validator.test.ts b/packages/cli/src/validators/__tests__/no-xss.validator.test.ts new file mode 100644 index 0000000000..75eaeb9072 --- /dev/null +++ b/packages/cli/src/validators/__tests__/no-xss.validator.test.ts @@ -0,0 +1,117 @@ +import { NoXss } from '../no-xss.validator'; +import { validate } from 'class-validator'; + +describe('NoXss', () => { + class Entity { + @NoXss() + name = ''; + + @NoXss() + timestamp = ''; + + @NoXss() + version = ''; + + @NoXss({ each: true }) + categories: string[] = []; + } + + const entity = new Entity(); + + describe('Scripts', () => { + // eslint-disable-next-line n8n-local-rules/no-unneeded-backticks + const XSS_STRINGS = ['", `Jack`]; + + for (const str of XSS_STRINGS) { + test(`should block ${str}`, async () => { + entity.name = str; + const errors = await validate(entity); + expect(errors).toHaveLength(1); + const [error] = errors; + expect(error.property).toEqual('name'); + expect(error.constraints).toEqual({ NoXss: 'Potentially malicious string' }); + }); + } + }); + + describe('Names', () => { + const VALID_NAMES = [ + 'Johann Strauß', + 'Вагиф Сәмәдоғлу', + 'René Magritte', + 'সুকুমার রায়', + 'མགོན་པོ་རྡོ་རྗེ།', + 'عبدالحليم حافظ', + ]; + + for (const name of VALID_NAMES) { + test(`should allow ${name}`, async () => { + entity.name = name; + expect(await validate(entity)).toBeEmptyArray(); + }); + } + }); + + describe('ISO-8601 timestamps', () => { + const VALID_TIMESTAMPS = ['2022-01-01T00:00:00.000Z', '2022-01-01T00:00:00.000+02:00']; + + for (const timestamp of VALID_TIMESTAMPS) { + test(`should allow ${timestamp}`, async () => { + entity.timestamp = timestamp; + await expect(validate(entity)).resolves.toBeEmptyArray(); + }); + } + }); + + describe('Semver versions', () => { + const VALID_VERSIONS = ['1.0.0', '1.0.0-alpha.1']; + + for (const version of VALID_VERSIONS) { + test(`should allow ${version}`, async () => { + entity.version = version; + await expect(validate(entity)).resolves.toBeEmptyArray(); + }); + } + }); + + describe('Miscellaneous strings', () => { + const VALID_MISCELLANEOUS_STRINGS = ['CI/CD']; + + for (const str of VALID_MISCELLANEOUS_STRINGS) { + test(`should allow ${str}`, async () => { + entity.name = str; + await expect(validate(entity)).resolves.toBeEmptyArray(); + }); + } + }); + + describe('Array of strings', () => { + const VALID_STRING_ARRAYS = [ + ['cloud-infrastructure-orchestration', 'ci-cd', 'reporting'], + ['automationGoalDevops', 'cloudComputing', 'containerization'], + ]; + + for (const arr of VALID_STRING_ARRAYS) { + test(`should allow array: ${JSON.stringify(arr)}`, async () => { + entity.categories = arr; + await expect(validate(entity)).resolves.toBeEmptyArray(); + }); + } + + const INVALID_STRING_ARRAYS = [ + ['valid-string', '', 'another-valid-string'], + ['', 'valid-string'], + ]; + + for (const arr of INVALID_STRING_ARRAYS) { + test(`should reject array containing invalid string: ${JSON.stringify(arr)}`, async () => { + entity.categories = arr; + const errors = await validate(entity); + expect(errors).toHaveLength(1); + const [error] = errors; + expect(error.property).toEqual('categories'); + expect(error.constraints).toEqual({ NoXss: 'Potentially malicious string' }); + }); + } + }); +}); diff --git a/packages/cli/src/validators/no-url.validator.ts b/packages/cli/src/validators/no-url.validator.ts new file mode 100644 index 0000000000..0cdacaddc1 --- /dev/null +++ b/packages/cli/src/validators/no-url.validator.ts @@ -0,0 +1,27 @@ +import type { ValidationOptions, ValidatorConstraintInterface } from 'class-validator'; +import { registerDecorator, ValidatorConstraint } from 'class-validator'; + +const URL_REGEX = /^(https?:\/\/|www\.)|(\.[\p{L}\d-]+)/iu; + +@ValidatorConstraint({ name: 'NoUrl', async: false }) +class NoUrlConstraint implements ValidatorConstraintInterface { + validate(value: string) { + return !URL_REGEX.test(value); + } + + defaultMessage() { + return 'Potentially malicious string'; + } +} + +export function NoUrl(options?: ValidationOptions) { + return function (object: object, propertyName: string) { + registerDecorator({ + name: 'NoUrl', + target: object.constructor, + propertyName, + options, + validator: NoUrlConstraint, + }); + }; +} diff --git a/packages/cli/src/validators/no-xss.validator.ts b/packages/cli/src/validators/no-xss.validator.ts new file mode 100644 index 0000000000..69960c39dd --- /dev/null +++ b/packages/cli/src/validators/no-xss.validator.ts @@ -0,0 +1,33 @@ +import xss from 'xss'; +import type { ValidationOptions, ValidatorConstraintInterface } from 'class-validator'; +import { registerDecorator, ValidatorConstraint } from 'class-validator'; + +@ValidatorConstraint({ name: 'NoXss', async: false }) +class NoXssConstraint implements ValidatorConstraintInterface { + validate(value: unknown) { + if (typeof value !== 'string') return false; + + return ( + value === + xss(value, { + whiteList: {}, // no tags are allowed + }) + ); + } + + defaultMessage() { + return 'Potentially malicious string'; + } +} + +export function NoXss(options?: ValidationOptions) { + return function (object: object, propertyName: string) { + registerDecorator({ + name: 'NoXss', + target: object.constructor, + propertyName, + options, + validator: NoXssConstraint, + }); + }; +} diff --git a/packages/cli/src/WaitTracker.ts b/packages/cli/src/wait-tracker.ts similarity index 98% rename from packages/cli/src/WaitTracker.ts rename to packages/cli/src/wait-tracker.ts index 5050d80ba9..eec9cb64b1 100644 --- a/packages/cli/src/WaitTracker.ts +++ b/packages/cli/src/wait-tracker.ts @@ -1,10 +1,10 @@ import { ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; import { Service } from 'typedi'; import type { IWorkflowExecutionDataProcess } from '@/Interfaces'; -import { WorkflowRunner } from '@/WorkflowRunner'; +import { WorkflowRunner } from '@/workflow-runner'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { OwnershipService } from '@/services/ownership.service'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { OrchestrationService } from '@/services/orchestration.service'; @Service() diff --git a/packages/cli/src/WaitingForms.ts b/packages/cli/src/waiting-forms.ts similarity index 89% rename from packages/cli/src/WaitingForms.ts rename to packages/cli/src/waiting-forms.ts index 0625acd7e4..2fc1655594 100644 --- a/packages/cli/src/WaitingForms.ts +++ b/packages/cli/src/waiting-forms.ts @@ -1,7 +1,7 @@ import { Service } from 'typedi'; import type { IExecutionResponse } from '@/Interfaces'; -import { WaitingWebhooks } from '@/WaitingWebhooks'; +import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; @Service() export class WaitingForms extends WaitingWebhooks { diff --git a/packages/cli/test/unit/TestWebhooks.test.ts b/packages/cli/src/webhooks/__tests__/TestWebhooks.test.ts similarity index 91% rename from packages/cli/test/unit/TestWebhooks.test.ts rename to packages/cli/src/webhooks/__tests__/TestWebhooks.test.ts index 6c7ae555b3..292b20e1bd 100644 --- a/packages/cli/test/unit/TestWebhooks.test.ts +++ b/packages/cli/src/webhooks/__tests__/TestWebhooks.test.ts @@ -1,22 +1,23 @@ import { mock } from 'jest-mock-extended'; -import { TestWebhooks } from '@/TestWebhooks'; +import { TestWebhooks } from '@/webhooks/test-webhooks'; import { WebhookNotFoundError } from '@/errors/response-errors/webhook-not-found.error'; import { v4 as uuid } from 'uuid'; import { generateNanoId } from '@/databases/utils/generators'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import * as WebhookHelpers from '@/WebhookHelpers'; +import * as WebhookHelpers from '@/webhooks/webhook-helpers'; import type * as express from 'express'; -import type { IWorkflowDb, WebhookRequest } from '@/Interfaces'; +import type { IWorkflowDb } from '@/Interfaces'; import type { IWebhookData, IWorkflowExecuteAdditionalData, Workflow } from 'n8n-workflow'; import type { TestWebhookRegistrationsService, TestWebhookRegistration, -} from '@/services/test-webhook-registrations.service'; +} from '@/webhooks/test-webhook-registrations.service'; -import * as AdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as AdditionalData from '@/workflow-execute-additional-data'; +import type { WebhookRequest } from '@/webhooks/webhook.types'; -jest.mock('@/WorkflowExecuteAdditionalData'); +jest.mock('@/workflow-execute-additional-data'); const mockedAdditionalData = AdditionalData as jest.Mocked; diff --git a/packages/cli/src/webhooks/__tests__/WebhookRequestHandler.test.ts b/packages/cli/src/webhooks/__tests__/WebhookRequestHandler.test.ts new file mode 100644 index 0000000000..c807c6c4d9 --- /dev/null +++ b/packages/cli/src/webhooks/__tests__/WebhookRequestHandler.test.ts @@ -0,0 +1,220 @@ +import { type Response } from 'express'; +import { mock } from 'jest-mock-extended'; +import { randomString } from 'n8n-workflow'; +import type { IHttpRequestMethods } from 'n8n-workflow'; + +import type { + IWebhookManager, + IWebhookResponseCallbackData, + WebhookOptionsRequest, + WebhookRequest, +} from '@/webhooks/webhook.types'; +import { createWebhookHandlerFor } from '@/webhooks/webhook-request-handler'; +import { ResponseError } from '@/errors/response-errors/abstract/response.error'; + +describe('WebhookRequestHandler', () => { + const webhookManager = mock>(); + const handler = createWebhookHandlerFor(webhookManager); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw for unsupported methods', async () => { + const req = mock({ + method: 'CONNECT' as IHttpRequestMethods, + }); + const res = mock(); + res.status.mockReturnValue(res); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(500); + expect(res.json).toHaveBeenCalledWith({ + code: 0, + message: 'The method CONNECT is not supported.', + }); + }); + + describe('preflight requests', () => { + it('should handle missing header for requested method', async () => { + const req = mock({ + method: 'OPTIONS', + headers: { + origin: 'https://example.com', + 'access-control-request-method': undefined, + }, + params: { path: 'test' }, + }); + const res = mock(); + res.status.mockReturnValue(res); + + webhookManager.getWebhookMethods.mockResolvedValue(['GET', 'PATCH']); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(204); + expect(res.header).toHaveBeenCalledWith( + 'Access-Control-Allow-Methods', + 'OPTIONS, GET, PATCH', + ); + }); + + it('should handle default origin and max-age', async () => { + const req = mock({ + method: 'OPTIONS', + headers: { + origin: 'https://example.com', + 'access-control-request-method': 'GET', + }, + params: { path: 'test' }, + }); + const res = mock(); + res.status.mockReturnValue(res); + + webhookManager.getWebhookMethods.mockResolvedValue(['GET', 'PATCH']); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(204); + expect(res.header).toHaveBeenCalledWith( + 'Access-Control-Allow-Methods', + 'OPTIONS, GET, PATCH', + ); + expect(res.header).toHaveBeenCalledWith('Access-Control-Allow-Origin', 'https://example.com'); + expect(res.header).toHaveBeenCalledWith('Access-Control-Max-Age', '300'); + }); + + it('should handle wildcard origin', async () => { + const randomOrigin = randomString(10); + const req = mock({ + method: 'OPTIONS', + headers: { + origin: randomOrigin, + 'access-control-request-method': 'GET', + }, + params: { path: 'test' }, + }); + const res = mock(); + res.status.mockReturnValue(res); + + webhookManager.getWebhookMethods.mockResolvedValue(['GET', 'PATCH']); + webhookManager.findAccessControlOptions.mockResolvedValue({ + allowedOrigins: '*', + }); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(204); + expect(res.header).toHaveBeenCalledWith( + 'Access-Control-Allow-Methods', + 'OPTIONS, GET, PATCH', + ); + expect(res.header).toHaveBeenCalledWith('Access-Control-Allow-Origin', randomOrigin); + }); + + it('should handle custom origin', async () => { + const req = mock({ + method: 'OPTIONS', + headers: { + origin: 'https://example.com', + 'access-control-request-method': 'GET', + }, + params: { path: 'test' }, + }); + const res = mock(); + res.status.mockReturnValue(res); + + webhookManager.getWebhookMethods.mockResolvedValue(['GET', 'PATCH']); + webhookManager.findAccessControlOptions.mockResolvedValue({ + allowedOrigins: 'https://test.com', + }); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(204); + expect(res.header).toHaveBeenCalledWith( + 'Access-Control-Allow-Methods', + 'OPTIONS, GET, PATCH', + ); + expect(res.header).toHaveBeenCalledWith('Access-Control-Allow-Origin', 'https://test.com'); + }); + }); + + describe('webhook requests', () => { + it('should delegate the request to the webhook manager and send the response', async () => { + const req = mock({ + method: 'GET', + params: { path: 'test' }, + }); + + const res = mock(); + + const executeWebhookResponse: IWebhookResponseCallbackData = { + responseCode: 200, + data: {}, + headers: { + 'x-custom-header': 'test', + }, + }; + webhookManager.executeWebhook.mockResolvedValueOnce(executeWebhookResponse); + + await handler(req, res); + + expect(webhookManager.executeWebhook).toHaveBeenCalledWith(req, res); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.header).toHaveBeenCalledWith({ + 'x-custom-header': 'test', + }); + expect(res.json).toHaveBeenCalledWith(executeWebhookResponse.data); + }); + + it('should send an error response if webhook execution throws', async () => { + class TestError extends ResponseError {} + const req = mock({ + method: 'GET', + params: { path: 'test' }, + }); + + const res = mock(); + res.status.mockReturnValue(res); + + webhookManager.executeWebhook.mockRejectedValueOnce( + new TestError('Test error', 500, 100, 'Test hint'), + ); + + await handler(req, res); + + expect(webhookManager.executeWebhook).toHaveBeenCalledWith(req, res); + expect(res.status).toHaveBeenCalledWith(500); + expect(res.json).toHaveBeenCalledWith({ + code: 100, + message: 'Test error', + hint: 'Test hint', + }); + }); + + test.each(['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT'])( + "should handle '%s' method", + async (method) => { + const req = mock({ + method, + params: { path: 'test' }, + }); + + const res = mock(); + + const executeWebhookResponse: IWebhookResponseCallbackData = { + responseCode: 200, + }; + webhookManager.executeWebhook.mockResolvedValueOnce(executeWebhookResponse); + + await handler(req, res); + + expect(webhookManager.executeWebhook).toHaveBeenCalledWith(req, res); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith(executeWebhookResponse.data); + }, + ); + }); +}); diff --git a/packages/cli/test/unit/services/test-webhook-registrations.service.test.ts b/packages/cli/src/webhooks/__tests__/test-webhook-registrations.service.test.ts similarity index 95% rename from packages/cli/test/unit/services/test-webhook-registrations.service.test.ts rename to packages/cli/src/webhooks/__tests__/test-webhook-registrations.service.test.ts index c93540938c..95502e3611 100644 --- a/packages/cli/test/unit/services/test-webhook-registrations.service.test.ts +++ b/packages/cli/src/webhooks/__tests__/test-webhook-registrations.service.test.ts @@ -1,7 +1,7 @@ import type { CacheService } from '@/services/cache/cache.service'; import type { OrchestrationService } from '@/services/orchestration.service'; -import type { TestWebhookRegistration } from '@/services/test-webhook-registrations.service'; -import { TestWebhookRegistrationsService } from '@/services/test-webhook-registrations.service'; +import type { TestWebhookRegistration } from '@/webhooks/test-webhook-registrations.service'; +import { TestWebhookRegistrationsService } from '@/webhooks/test-webhook-registrations.service'; import { mock } from 'jest-mock-extended'; describe('TestWebhookRegistrationsService', () => { diff --git a/packages/cli/src/__tests__/waiting-webhooks.test.ts b/packages/cli/src/webhooks/__tests__/waiting-webhooks.test.ts similarity index 90% rename from packages/cli/src/__tests__/waiting-webhooks.test.ts rename to packages/cli/src/webhooks/__tests__/waiting-webhooks.test.ts index 6748c72335..5cb8e005ad 100644 --- a/packages/cli/src/__tests__/waiting-webhooks.test.ts +++ b/packages/cli/src/webhooks/__tests__/waiting-webhooks.test.ts @@ -1,10 +1,11 @@ import { mock } from 'jest-mock-extended'; -import { WaitingWebhooks } from '@/WaitingWebhooks'; +import { WaitingWebhooks } from '@/webhooks/waiting-webhooks'; import { ConflictError } from '@/errors/response-errors/conflict.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import type { IExecutionResponse, WaitingWebhookRequest } from '@/Interfaces'; +import type { IExecutionResponse } from '@/Interfaces'; import type express from 'express'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { WaitingWebhookRequest } from '@/webhooks/webhook.types'; describe('WaitingWebhooks', () => { const executionRepository = mock(); diff --git a/packages/cli/test/unit/services/webhook.service.test.ts b/packages/cli/src/webhooks/__tests__/webhook.service.test.ts similarity index 98% rename from packages/cli/test/unit/services/webhook.service.test.ts rename to packages/cli/src/webhooks/__tests__/webhook.service.test.ts index adb18de4b3..5a8e19e84c 100644 --- a/packages/cli/test/unit/services/webhook.service.test.ts +++ b/packages/cli/src/webhooks/__tests__/webhook.service.test.ts @@ -2,9 +2,9 @@ import { v4 as uuid } from 'uuid'; import config from '@/config'; import { WebhookRepository } from '@db/repositories/webhook.repository'; import { CacheService } from '@/services/cache/cache.service'; -import { WebhookService } from '@/services/webhook.service'; +import { WebhookService } from '@/webhooks/webhook.service'; import { WebhookEntity } from '@db/entities/WebhookEntity'; -import { mockInstance } from '../../shared/mocking'; +import { mockInstance } from '@test/mocking'; const createWebhook = (method: string, path: string, webhookId?: string, pathSegments?: number) => Object.assign(new WebhookEntity(), { diff --git a/packages/cli/src/ActiveWebhooks.ts b/packages/cli/src/webhooks/live-webhooks.ts similarity index 85% rename from packages/cli/src/ActiveWebhooks.ts rename to packages/cli/src/webhooks/live-webhooks.ts index 79626df025..1db85e2315 100644 --- a/packages/cli/src/ActiveWebhooks.ts +++ b/packages/cli/src/webhooks/live-webhooks.ts @@ -5,22 +5,27 @@ import type { INode, IWebhookData, IHttpRequestMethods } from 'n8n-workflow'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import type { - IResponseCallbackData, + IWebhookResponseCallbackData, IWebhookManager, WebhookAccessControlOptions, WebhookRequest, -} from '@/Interfaces'; -import { Logger } from '@/Logger'; -import { NodeTypes } from '@/NodeTypes'; -import { WebhookService } from '@/services/webhook.service'; +} from './webhook.types'; +import { Logger } from '@/logger'; +import { NodeTypes } from '@/node-types'; +import { WebhookService } from '@/webhooks/webhook.service'; import { WebhookNotFoundError } from '@/errors/response-errors/webhook-not-found.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; -import * as WebhookHelpers from '@/WebhookHelpers'; -import { WorkflowStaticDataService } from '@/workflows/workflowStaticData.service'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import * as WebhookHelpers from '@/webhooks/webhook-helpers'; +import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; +/** + * Service for handling the execution of live webhooks, i.e. webhooks + * that belong to activated workflows and use the production URL + * (https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.webhook/#webhook-urls) + */ @Service() -export class ActiveWebhooks implements IWebhookManager { +export class LiveWebhooks implements IWebhookManager { constructor( private readonly logger: Logger, private readonly nodeTypes: NodeTypes, @@ -57,7 +62,7 @@ export class ActiveWebhooks implements IWebhookManager { async executeWebhook( request: WebhookRequest, response: Response, - ): Promise { + ): Promise { const httpMethod = request.method; const path = request.params.path; @@ -72,11 +77,9 @@ export class ActiveWebhooks implements IWebhookManager { const pathElements = path.split('/').slice(1); // extracting params from path - // @ts-ignore webhook.webhookPath.split('/').forEach((ele, index) => { if (ele.startsWith(':')) { // write params to req.params - // @ts-ignore request.params[ele.slice(1)] = pathElements[index]; } }); diff --git a/packages/cli/src/services/test-webhook-registrations.service.ts b/packages/cli/src/webhooks/test-webhook-registrations.service.ts similarity index 97% rename from packages/cli/src/services/test-webhook-registrations.service.ts rename to packages/cli/src/webhooks/test-webhook-registrations.service.ts index e2abae5605..94e7e7d826 100644 --- a/packages/cli/src/services/test-webhook-registrations.service.ts +++ b/packages/cli/src/webhooks/test-webhook-registrations.service.ts @@ -3,7 +3,7 @@ import { CacheService } from '@/services/cache/cache.service'; import type { IWebhookData } from 'n8n-workflow'; import type { IWorkflowDb } from '@/Interfaces'; import { TEST_WEBHOOK_TIMEOUT, TEST_WEBHOOK_TIMEOUT_BUFFER } from '@/constants'; -import { OrchestrationService } from './orchestration.service'; +import { OrchestrationService } from '@/services/orchestration.service'; export type TestWebhookRegistration = { pushRef?: string; diff --git a/packages/cli/src/TestWebhooks.ts b/packages/cli/src/webhooks/test-webhooks.ts similarity index 93% rename from packages/cli/src/TestWebhooks.ts rename to packages/cli/src/webhooks/test-webhooks.ts index 827226ee54..e10b77aa24 100644 --- a/packages/cli/src/TestWebhooks.ts +++ b/packages/cli/src/webhooks/test-webhooks.ts @@ -8,26 +8,30 @@ import type { IRunData, } from 'n8n-workflow'; import type { - IResponseCallbackData, + IWebhookResponseCallbackData, IWebhookManager, - IWorkflowDb, WebhookAccessControlOptions, WebhookRequest, -} from '@/Interfaces'; +} from './webhook.types'; import { Push } from '@/push'; -import { NodeTypes } from '@/NodeTypes'; -import * as WebhookHelpers from '@/WebhookHelpers'; +import { NodeTypes } from '@/node-types'; +import * as WebhookHelpers from '@/webhooks/webhook-helpers'; import { TEST_WEBHOOK_TIMEOUT } from '@/constants'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { WorkflowMissingIdError } from '@/errors/workflow-missing-id.error'; import { WebhookNotFoundError } from '@/errors/response-errors/webhook-not-found.error'; import * as NodeExecuteFunctions from 'n8n-core'; -import { removeTrailingSlash } from './utils'; -import type { TestWebhookRegistration } from '@/services/test-webhook-registrations.service'; -import { TestWebhookRegistrationsService } from '@/services/test-webhook-registrations.service'; +import { removeTrailingSlash } from '@/utils'; +import type { TestWebhookRegistration } from '@/webhooks/test-webhook-registrations.service'; +import { TestWebhookRegistrationsService } from '@/webhooks/test-webhook-registrations.service'; import { OrchestrationService } from '@/services/orchestration.service'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import type { IWorkflowDb } from '@/Interfaces'; +/** + * Service for handling the execution of webhooks of manual executions + * that use the [Test URL](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.webhook/#webhook-urls). + */ @Service() export class TestWebhooks implements IWebhookManager { constructor( @@ -46,7 +50,7 @@ export class TestWebhooks implements IWebhookManager { async executeWebhook( request: WebhookRequest, response: express.Response, - ): Promise { + ): Promise { const httpMethod = request.method; let path = removeTrailingSlash(request.params.path); @@ -117,7 +121,7 @@ export class TestWebhooks implements IWebhookManager { undefined, // executionId request, response, - (error: Error | null, data: IResponseCallbackData) => { + (error: Error | null, data: IWebhookResponseCallbackData) => { if (error !== null) reject(error); else resolve(data); }, diff --git a/packages/cli/src/WaitingWebhooks.ts b/packages/cli/src/webhooks/waiting-webhooks.ts similarity index 84% rename from packages/cli/src/WaitingWebhooks.ts rename to packages/cli/src/webhooks/waiting-webhooks.ts index d795c948f1..f2c5706817 100644 --- a/packages/cli/src/WaitingWebhooks.ts +++ b/packages/cli/src/webhooks/waiting-webhooks.ts @@ -2,21 +2,25 @@ import { NodeHelpers, Workflow } from 'n8n-workflow'; import { Service } from 'typedi'; import type express from 'express'; -import * as WebhookHelpers from '@/WebhookHelpers'; -import { NodeTypes } from '@/NodeTypes'; +import * as WebhookHelpers from '@/webhooks/webhook-helpers'; +import { NodeTypes } from '@/node-types'; import type { - IExecutionResponse, - IResponseCallbackData, + IWebhookResponseCallbackData, IWebhookManager, - IWorkflowDb, WaitingWebhookRequest, -} from '@/Interfaces'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; +} from './webhook.types'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { Logger } from '@/Logger'; -import { ConflictError } from './errors/response-errors/conflict.error'; -import { NotFoundError } from './errors/response-errors/not-found.error'; +import { Logger } from '@/logger'; +import { ConflictError } from '@/errors/response-errors/conflict.error'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import type { IExecutionResponse, IWorkflowDb } from '@/Interfaces'; +/** + * Service for handling the execution of webhooks of Wait nodes that use the + * [Resume On Webhook Call](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.wait/#on-webhook-call) + * feature. + */ @Service() export class WaitingWebhooks implements IWebhookManager { protected includeForms = false; @@ -40,7 +44,7 @@ export class WaitingWebhooks implements IWebhookManager { async executeWebhook( req: WaitingWebhookRequest, res: express.Response, - ): Promise { + ): Promise { const { path: executionId, suffix } = req.params; this.logReceivedWebhook(req.method, executionId); diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/webhooks/webhook-helpers.ts similarity index 83% rename from packages/cli/src/WebhookHelpers.ts rename to packages/cli/src/webhooks/webhook-helpers.ts index 95e8825e1a..725f0339a1 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/webhooks/webhook-helpers.ts @@ -20,8 +20,6 @@ import type { IDataObject, IDeferredPromise, IExecuteData, - IExecuteResponsePromiseData, - IHttpRequestMethods, IN8nHttpFullResponse, INode, IPinData, @@ -42,119 +40,20 @@ import { NodeHelpers, } from 'n8n-workflow'; -import type { - IExecutionDb, - IResponseCallbackData, - IWebhookManager, - IWorkflowDb, - IWorkflowExecutionDataProcess, - WebhookCORSRequest, - WebhookRequest, -} from '@/Interfaces'; -import * as ResponseHelper from '@/ResponseHelper'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; -import { WorkflowRunner } from '@/WorkflowRunner'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; -import { ActiveExecutions } from '@/ActiveExecutions'; +import type { IWebhookResponseCallbackData, WebhookRequest } from './webhook.types'; +import * as WorkflowHelpers from '@/workflow-helpers'; +import { WorkflowRunner } from '@/workflow-runner'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import { ActiveExecutions } from '@/active-executions'; import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; -import { OwnershipService } from './services/ownership.service'; -import { parseBody } from './middlewares'; -import { Logger } from './Logger'; -import { NotFoundError } from './errors/response-errors/not-found.error'; -import { InternalServerError } from './errors/response-errors/internal-server.error'; -import { UnprocessableRequestError } from './errors/response-errors/unprocessable.error'; -import type { Project } from './databases/entities/Project'; - -export const WEBHOOK_METHODS: IHttpRequestMethods[] = [ - 'DELETE', - 'GET', - 'HEAD', - 'PATCH', - 'POST', - 'PUT', -]; - -export const webhookRequestHandler = - (webhookManager: IWebhookManager) => - async (req: WebhookRequest | WebhookCORSRequest, res: express.Response) => { - const { path } = req.params; - const method = req.method; - - if (method !== 'OPTIONS' && !WEBHOOK_METHODS.includes(method)) { - return ResponseHelper.sendErrorResponse( - res, - new Error(`The method ${method} is not supported.`), - ); - } - - // Setup CORS headers only if the incoming request has an `origin` header - if ('origin' in req.headers) { - if (webhookManager.getWebhookMethods) { - try { - const allowedMethods = await webhookManager.getWebhookMethods(path); - res.header('Access-Control-Allow-Methods', ['OPTIONS', ...allowedMethods].join(', ')); - } catch (error) { - return ResponseHelper.sendErrorResponse(res, error as Error); - } - } - - const requestedMethod = - method === 'OPTIONS' - ? (req.headers['access-control-request-method'] as IHttpRequestMethods) - : method; - if (webhookManager.findAccessControlOptions && requestedMethod) { - const options = await webhookManager.findAccessControlOptions(path, requestedMethod); - const { allowedOrigins } = options ?? {}; - - if (allowedOrigins && allowedOrigins !== '*' && allowedOrigins !== req.headers.origin) { - const originsList = allowedOrigins.split(','); - const defaultOrigin = originsList[0]; - - if (originsList.length === 1) { - res.header('Access-Control-Allow-Origin', defaultOrigin); - } - - if (originsList.includes(req.headers.origin as string)) { - res.header('Access-Control-Allow-Origin', req.headers.origin); - } else { - res.header('Access-Control-Allow-Origin', defaultOrigin); - } - } else { - res.header('Access-Control-Allow-Origin', req.headers.origin); - } - - if (method === 'OPTIONS') { - res.header('Access-Control-Max-Age', '300'); - const requestedHeaders = req.headers['access-control-request-headers']; - if (requestedHeaders?.length) { - res.header('Access-Control-Allow-Headers', requestedHeaders); - } - } - } - } - - if (method === 'OPTIONS') { - return ResponseHelper.sendSuccessResponse(res, {}, true, 204); - } - - let response; - try { - response = await webhookManager.executeWebhook(req, res); - } catch (error) { - return ResponseHelper.sendErrorResponse(res, error as Error); - } - - // Don't respond, if already responded - if (response.noWebhookResponse !== true) { - ResponseHelper.sendSuccessResponse( - res, - response.data, - true, - response.responseCode, - response.headers, - ); - } - }; +import { OwnershipService } from '@/services/ownership.service'; +import { parseBody } from '@/middlewares'; +import { Logger } from '@/logger'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import { InternalServerError } from '@/errors/response-errors/internal-server.error'; +import { UnprocessableRequestError } from '@/errors/response-errors/unprocessable.error'; +import type { Project } from '@/databases/entities/Project'; +import type { IExecutionDb, IWorkflowDb, IWorkflowExecutionDataProcess } from '@/Interfaces'; /** * Returns all the webhooks which should be created for the given workflow @@ -192,18 +91,6 @@ export function getWorkflowWebhooks( return returnData; } -export function encodeWebhookResponse( - response: IExecuteResponsePromiseData, -): IExecuteResponsePromiseData { - if (typeof response === 'object' && Buffer.isBuffer(response.body)) { - response.body = { - '__@N8nEncodedBuffer@__': response.body.toString(BINARY_ENCODING), - }; - } - - return response; -} - const normalizeFormData = (values: Record) => { for (const key in values) { const value = values[key]; @@ -228,7 +115,7 @@ export async function executeWebhook( executionId: string | undefined, req: WebhookRequest, res: express.Response, - responseCallback: (error: Error | null, data: IResponseCallbackData) => void, + responseCallback: (error: Error | null, data: IWebhookResponseCallbackData) => void, destinationNode?: string, ): Promise { // Get the nodeType to know which responseMode is set @@ -256,6 +143,10 @@ export async function executeWebhook( // Prepare everything that is needed to run the workflow const additionalData = await WorkflowExecuteAdditionalData.getBase(); + if (executionId) { + additionalData.executionId = executionId; + } + // Get the responseMode const responseMode = workflow.expression.getSimpleParameterValue( workflowStartNode, @@ -359,12 +250,12 @@ export async function executeWebhook( additionalData, NodeExecuteFunctions, executionMode, + runExecutionData ?? null, ); - Container.get(WorkflowStatisticsService).emit( - 'nodeFetchedData', - workflow.id, - workflowStartNode, - ); + Container.get(WorkflowStatisticsService).emit('nodeFetchedData', { + workflowId: workflow.id, + node: workflowStartNode, + }); } catch (err) { // Send error response to webhook caller const errorMessage = 'Workflow Webhook Error: Workflow could not be started!'; diff --git a/packages/cli/src/webhooks/webhook-request-handler.ts b/packages/cli/src/webhooks/webhook-request-handler.ts new file mode 100644 index 0000000000..a495873a68 --- /dev/null +++ b/packages/cli/src/webhooks/webhook-request-handler.ts @@ -0,0 +1,119 @@ +import type express from 'express'; +import type { IHttpRequestMethods } from 'n8n-workflow'; +import type { + IWebhookManager, + WebhookOptionsRequest, + WebhookRequest, +} from '@/webhooks/webhook.types'; +import * as ResponseHelper from '@/response-helper'; + +const WEBHOOK_METHODS: IHttpRequestMethods[] = ['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT']; + +class WebhookRequestHandler { + constructor(private readonly webhookManager: IWebhookManager) {} + + /** + * Handles an incoming webhook request. Handles CORS and delegates the + * request to the webhook manager to execute the webhook. + */ + async handleRequest(req: WebhookRequest | WebhookOptionsRequest, res: express.Response) { + const method = req.method; + + if (method !== 'OPTIONS' && !WEBHOOK_METHODS.includes(method)) { + return ResponseHelper.sendErrorResponse( + res, + new Error(`The method ${method} is not supported.`), + ); + } + + // Setup CORS headers only if the incoming request has an `origin` header + if ('origin' in req.headers) { + const corsSetupError = await this.setupCorsHeaders(req, res); + if (corsSetupError) { + return ResponseHelper.sendErrorResponse(res, corsSetupError); + } + } + + if (method === 'OPTIONS') { + return ResponseHelper.sendSuccessResponse(res, {}, true, 204); + } + + try { + const response = await this.webhookManager.executeWebhook(req, res); + + // Don't respond, if already responded + if (response.noWebhookResponse !== true) { + ResponseHelper.sendSuccessResponse( + res, + response.data, + true, + response.responseCode, + response.headers, + ); + } + } catch (error) { + return ResponseHelper.sendErrorResponse(res, error as Error); + } + } + + private async setupCorsHeaders( + req: WebhookRequest | WebhookOptionsRequest, + res: express.Response, + ): Promise { + const method = req.method; + const { path } = req.params; + + if (this.webhookManager.getWebhookMethods) { + try { + const allowedMethods = await this.webhookManager.getWebhookMethods(path); + res.header('Access-Control-Allow-Methods', ['OPTIONS', ...allowedMethods].join(', ')); + } catch (error) { + return error as Error; + } + } + + const requestedMethod = + method === 'OPTIONS' + ? (req.headers['access-control-request-method'] as IHttpRequestMethods) + : method; + if (this.webhookManager.findAccessControlOptions && requestedMethod) { + const options = await this.webhookManager.findAccessControlOptions(path, requestedMethod); + const { allowedOrigins } = options ?? {}; + + if (allowedOrigins && allowedOrigins !== '*' && allowedOrigins !== req.headers.origin) { + const originsList = allowedOrigins.split(','); + const defaultOrigin = originsList[0]; + + if (originsList.length === 1) { + res.header('Access-Control-Allow-Origin', defaultOrigin); + } + + if (originsList.includes(req.headers.origin as string)) { + res.header('Access-Control-Allow-Origin', req.headers.origin); + } else { + res.header('Access-Control-Allow-Origin', defaultOrigin); + } + } else { + res.header('Access-Control-Allow-Origin', req.headers.origin); + } + + if (method === 'OPTIONS') { + res.header('Access-Control-Max-Age', '300'); + const requestedHeaders = req.headers['access-control-request-headers']; + if (requestedHeaders?.length) { + res.header('Access-Control-Allow-Headers', requestedHeaders); + } + } + } + + return null; + } +} + +export function createWebhookHandlerFor(webhookManager: IWebhookManager) { + const handler = new WebhookRequestHandler(webhookManager); + + return async (req: WebhookRequest | WebhookOptionsRequest, res: express.Response) => { + await handler.handleRequest(req, res); + }; +} diff --git a/packages/cli/src/WebhookServer.ts b/packages/cli/src/webhooks/webhook-server.ts similarity index 72% rename from packages/cli/src/WebhookServer.ts rename to packages/cli/src/webhooks/webhook-server.ts index 60f59f606d..bb00510e56 100644 --- a/packages/cli/src/WebhookServer.ts +++ b/packages/cli/src/webhooks/webhook-server.ts @@ -1,5 +1,5 @@ import { Service } from 'typedi'; -import { AbstractServer } from '@/AbstractServer'; +import { AbstractServer } from '@/abstract-server'; @Service() export class WebhookServer extends AbstractServer { diff --git a/packages/cli/src/services/webhook.service.ts b/packages/cli/src/webhooks/webhook.service.ts similarity index 100% rename from packages/cli/src/services/webhook.service.ts rename to packages/cli/src/webhooks/webhook.service.ts diff --git a/packages/cli/src/webhooks/webhook.types.ts b/packages/cli/src/webhooks/webhook.types.ts new file mode 100644 index 0000000000..4d34b00d04 --- /dev/null +++ b/packages/cli/src/webhooks/webhook.types.ts @@ -0,0 +1,37 @@ +import type { Request, Response } from 'express'; +import type { IDataObject, IHttpRequestMethods } from 'n8n-workflow'; + +export type WebhookOptionsRequest = Request & { method: 'OPTIONS' }; + +export type WebhookRequest = Request<{ path: string }> & { + method: IHttpRequestMethods; + params: Record; +}; + +export type WaitingWebhookRequest = WebhookRequest & { + params: WebhookRequest['path'] & { suffix?: string }; +}; + +export interface WebhookAccessControlOptions { + allowedOrigins?: string; +} + +export interface IWebhookManager { + /** Gets all request methods associated with a webhook path*/ + getWebhookMethods?: (path: string) => Promise; + + /** Find the CORS options matching a path and method */ + findAccessControlOptions?: ( + path: string, + httpMethod: IHttpRequestMethods, + ) => Promise; + + executeWebhook(req: WebhookRequest, res: Response): Promise; +} + +export interface IWebhookResponseCallbackData { + data?: IDataObject | IDataObject[]; + headers?: object; + noWebhookResponse?: boolean; + responseCode?: number; +} diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/workflow-execute-additional-data.ts similarity index 93% rename from packages/cli/src/WorkflowExecuteAdditionalData.ts rename to packages/cli/src/workflow-execute-additional-data.ts index e5a17de7ec..459c946926 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/workflow-execute-additional-data.ts @@ -36,9 +36,9 @@ import { import { Container } from 'typedi'; import config from '@/config'; -import { ActiveExecutions } from '@/ActiveExecutions'; -import { CredentialsHelper } from '@/CredentialsHelper'; -import { ExternalHooks } from '@/ExternalHooks'; +import { ActiveExecutions } from '@/active-executions'; +import { CredentialsHelper } from '@/credentials-helper'; +import { ExternalHooks } from '@/external-hooks'; import type { IPushDataExecutionFinished, IWorkflowExecuteProcess, @@ -47,31 +47,30 @@ import type { IPushDataType, ExecutionPayload, } from '@/Interfaces'; -import { NodeTypes } from '@/NodeTypes'; +import { NodeTypes } from '@/node-types'; import { Push } from '@/push'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; +import * as WorkflowHelpers from '@/workflow-helpers'; import { findSubworkflowStart, isWorkflowIdValid } from '@/utils'; -import { PermissionChecker } from './UserManagement/PermissionChecker'; -import { InternalHooks } from '@/InternalHooks'; +import { PermissionChecker } from './user-management/permission-checker'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; -import { SecretsHelper } from './SecretsHelpers'; +import { SecretsHelper } from './secrets-helpers'; import { OwnershipService } from './services/ownership.service'; import { determineFinalExecutionStatus, prepareExecutionDataForDbUpdate, updateExistingExecution, -} from './executionLifecycleHooks/shared/sharedHookFunctions'; -import { restoreBinaryDataId } from './executionLifecycleHooks/restoreBinaryDataId'; -import { toSaveSettings } from './executionLifecycleHooks/toSaveSettings'; -import { Logger } from './Logger'; -import { saveExecutionProgress } from './executionLifecycleHooks/saveExecutionProgress'; -import { WorkflowStaticDataService } from './workflows/workflowStaticData.service'; +} from './execution-lifecycle-hooks/shared/shared-hook-functions'; +import { restoreBinaryDataId } from './execution-lifecycle-hooks/restore-binary-data-id'; +import { toSaveSettings } from './execution-lifecycle-hooks/to-save-settings'; +import { Logger } from './logger'; +import { saveExecutionProgress } from './execution-lifecycle-hooks/save-execution-progress'; +import { WorkflowStaticDataService } from './workflows/workflow-static-data.service'; import { WorkflowRepository } from './databases/repositories/workflow.repository'; import { UrlService } from './services/url.service'; -import { WorkflowExecutionService } from './workflows/workflowExecution.service'; +import { WorkflowExecutionService } from './workflows/workflow-execution.service'; import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; -import { EventService } from './eventbus/event.service'; +import { EventService } from './events/event.service'; import { GlobalConfig } from '@n8n/config'; import { SubworkflowPolicyChecker } from './subworkflows/subworkflow-policy-checker.service'; @@ -525,17 +524,16 @@ function hookFunctionsSave(): IWorkflowExecuteHooks { ); } } finally { - workflowStatisticsService.emit( - 'workflowExecutionCompleted', - this.workflowData, + workflowStatisticsService.emit('workflowExecutionCompleted', { + workflowData: this.workflowData, fullRunData, - ); + }); } }, ], nodeFetchedData: [ async (workflowId: string, node: INode) => { - workflowStatisticsService.emit('nodeFetchedData', workflowId, node); + workflowStatisticsService.emit('nodeFetchedData', { workflowId, node }); }, ], }; @@ -549,7 +547,6 @@ function hookFunctionsSave(): IWorkflowExecuteHooks { */ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { const logger = Container.get(Logger); - const internalHooks = Container.get(InternalHooks); const workflowStatisticsService = Container.get(WorkflowStatisticsService); const eventService = Container.get(EventService); return { @@ -636,23 +633,19 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { this.retryOf, ); } finally { - workflowStatisticsService.emit( - 'workflowExecutionCompleted', - this.workflowData, + workflowStatisticsService.emit('workflowExecutionCompleted', { + workflowData: this.workflowData, fullRunData, - ); + }); } }, async function (this: WorkflowHooks, runData: IRun): Promise { const { executionId, workflowData: workflow } = this; - void internalHooks.onWorkflowPostExecute(executionId, workflow, runData); eventService.emit('workflow-post-execute', { - workflowId: workflow.id, - workflowName: workflow.name, + workflow, executionId, - success: runData.status === 'success', - isManual: runData.mode === 'manual', + runData, }); }, async function (this: WorkflowHooks, fullRunData: IRun) { @@ -676,7 +669,7 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { ], nodeFetchedData: [ async (workflowId: string, node: INode) => { - workflowStatisticsService.emit('nodeFetchedData', workflowId, node); + workflowStatisticsService.emit('nodeFetchedData', { workflowId, node }); }, ], }; @@ -724,7 +717,6 @@ export async function getRunData( const runData: IWorkflowExecutionDataProcess = { executionMode: mode, executionData: runExecutionData, - // @ts-ignore workflowData, }; @@ -788,7 +780,6 @@ async function executeWorkflow( parentCallbackManager?: CallbackManager; }, ): Promise | IWorkflowExecuteProcess> { - const internalHooks = Container.get(InternalHooks); const externalHooks = Container.get(ExternalHooks); await externalHooks.init(); @@ -934,14 +925,11 @@ async function executeWorkflow( await externalHooks.run('workflow.postExecute', [data, workflowData, executionId]); - void internalHooks.onWorkflowPostExecute(executionId, workflowData, data, additionalData.userId); eventService.emit('workflow-post-execute', { - workflowId: workflowData.id, - workflowName: workflowData.name, + workflow: workflowData, executionId, - success: data.status === 'success', - isManual: data.mode === 'manual', userId: additionalData.userId, + runData: data, }); // subworkflow either finished, or is in status waiting due to a wait node, both cases are considered successes here @@ -1002,23 +990,19 @@ export async function getBase( ): Promise { const urlBaseWebhook = Container.get(UrlService).getWebhookBaseUrl(); - const formWaitingBaseUrl = urlBaseWebhook + config.getEnv('endpoints.formWaiting'); - - const webhookBaseUrl = urlBaseWebhook + config.getEnv('endpoints.webhook'); - const webhookTestBaseUrl = urlBaseWebhook + config.getEnv('endpoints.webhookTest'); - const webhookWaitingBaseUrl = urlBaseWebhook + config.getEnv('endpoints.webhookWaiting'); + const globalConfig = Container.get(GlobalConfig); const variables = await WorkflowHelpers.getVariables(); return { credentialsHelper: Container.get(CredentialsHelper), executeWorkflow, - restApiUrl: urlBaseWebhook + config.getEnv('endpoints.rest'), + restApiUrl: urlBaseWebhook + globalConfig.endpoints.rest, instanceBaseUrl: urlBaseWebhook, - formWaitingBaseUrl, - webhookBaseUrl, - webhookWaitingBaseUrl, - webhookTestBaseUrl, + formWaitingBaseUrl: urlBaseWebhook + globalConfig.endpoints.formWaiting, + webhookBaseUrl: urlBaseWebhook + globalConfig.endpoints.webhook, + webhookWaitingBaseUrl: urlBaseWebhook + globalConfig.endpoints.webhookWaiting, + webhookTestBaseUrl: urlBaseWebhook + globalConfig.endpoints.webhookTest, currentNodeParameters, executionTimeoutTimestamp, userId, diff --git a/packages/cli/src/WorkflowHelpers.ts b/packages/cli/src/workflow-helpers.ts similarity index 100% rename from packages/cli/src/WorkflowHelpers.ts rename to packages/cli/src/workflow-helpers.ts diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/workflow-runner.ts similarity index 90% rename from packages/cli/src/WorkflowRunner.ts rename to packages/cli/src/workflow-runner.ts index f8faf55fbe..b8012b3d4f 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -22,26 +22,26 @@ import { import PCancelable from 'p-cancelable'; -import { ActiveExecutions } from '@/ActiveExecutions'; +import { ActiveExecutions } from '@/active-executions'; import config from '@/config'; import { ExecutionRepository } from '@db/repositories/execution.repository'; -import { ExternalHooks } from '@/ExternalHooks'; +import { ExternalHooks } from '@/external-hooks'; import type { IExecutionResponse, IWorkflowExecutionDataProcess } from '@/Interfaces'; -import { NodeTypes } from '@/NodeTypes'; -import type { Job, JobData, JobResponse } from '@/Queue'; -import { Queue } from '@/Queue'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; -import { generateFailedExecutionFromError } from '@/WorkflowHelpers'; -import { PermissionChecker } from '@/UserManagement/PermissionChecker'; -import { InternalHooks } from '@/InternalHooks'; -import { Logger } from '@/Logger'; -import { WorkflowStaticDataService } from '@/workflows/workflowStaticData.service'; -import { EventService } from './eventbus/event.service'; +import { NodeTypes } from '@/node-types'; +import type { Job, JobData, JobResult } from '@/scaling/types'; +import type { ScalingService } from '@/scaling/scaling.service'; +import * as WorkflowHelpers from '@/workflow-helpers'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import { generateFailedExecutionFromError } from '@/workflow-helpers'; +import { PermissionChecker } from '@/user-management/permission-checker'; +import { Logger } from '@/logger'; +import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; +import { EventService } from './events/event.service'; +import { GlobalConfig } from '@n8n/config'; @Service() export class WorkflowRunner { - private jobQueue: Queue; + private scalingService: ScalingService; private executionsMode = config.getEnv('executions.mode'); @@ -54,11 +54,7 @@ export class WorkflowRunner { private readonly nodeTypes: NodeTypes, private readonly permissionChecker: PermissionChecker, private readonly eventService: EventService, - ) { - if (this.executionsMode === 'queue') { - this.jobQueue = Container.get(Queue); - } - } + ) {} /** The process did error */ async processError( @@ -160,19 +156,11 @@ export class WorkflowRunner { const postExecutePromise = this.activeExecutions.getPostExecutePromise(executionId); postExecutePromise .then(async (executionData) => { - void Container.get(InternalHooks).onWorkflowPostExecute( - executionId, - data.workflowData, - executionData, - data.userId, - ); this.eventService.emit('workflow-post-execute', { - workflowId: data.workflowData.id, - workflowName: data.workflowData.name, + workflow: data.workflowData, executionId, - success: executionData?.status === 'success', - isManual: data.executionMode === 'manual', userId: data.userId, + runData: executionData, }); if (this.externalHooks.exists('workflow.postExecute')) { try { @@ -369,6 +357,11 @@ export class WorkflowRunner { loadStaticData: !!loadStaticData, }; + if (!this.scalingService) { + const { ScalingService } = await import('@/scaling/scaling.service'); + this.scalingService = Container.get(ScalingService); + } + let priority = 100; if (realtime === true) { // Jobs which require a direct response get a higher priority @@ -384,9 +377,7 @@ export class WorkflowRunner { let job: Job; let hooks: WorkflowHooks; try { - job = await this.jobQueue.add(jobData, jobOptions); - - this.logger.info(`Started with job ID: ${job.id.toString()} (Execution ID: ${executionId})`); + job = await this.scalingService.addJob(jobData, jobOptions); hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerMain( data.executionMode, @@ -415,8 +406,7 @@ export class WorkflowRunner { async (resolve, reject, onCancel) => { onCancel.shouldReject = false; onCancel(async () => { - const queue = Container.get(Queue); - await queue.stopJob(job); + await this.scalingService.stopJob(job); // We use "getWorkflowHooksWorkerExecuter" as "getWorkflowHooksWorkerMain" does not contain the // "workflowExecuteAfter" which we require. @@ -433,11 +423,11 @@ export class WorkflowRunner { reject(error); }); - const jobData: Promise = job.finished(); + const jobData: Promise = job.finished(); - const queueRecoveryInterval = config.getEnv('queue.bull.queueRecoveryInterval'); + const { queueRecoveryInterval } = Container.get(GlobalConfig).queue.bull; - const racingPromises: Array> = [jobData]; + const racingPromises: Array> = [jobData]; let clearWatchdogInterval; if (queueRecoveryInterval > 0) { @@ -455,9 +445,9 @@ export class WorkflowRunner { ************************************************ */ let watchDogInterval: NodeJS.Timeout | undefined; - const watchDog: Promise = new Promise((res) => { + const watchDog: Promise = new Promise((res) => { watchDogInterval = setInterval(async () => { - const currentJob = await this.jobQueue.getJob(job.id); + const currentJob = await this.scalingService.getJob(job.id); // When null means job is finished (not found in queue) if (currentJob === null) { // Mimic worker's success message diff --git a/packages/cli/test/unit/workflow-execution.service.test.ts b/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts similarity index 96% rename from packages/cli/test/unit/workflow-execution.service.test.ts rename to packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts index d06dde35e2..b56cbffbbd 100644 --- a/packages/cli/test/unit/workflow-execution.service.test.ts +++ b/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts @@ -3,8 +3,8 @@ import { mock } from 'jest-mock-extended'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import type { IWorkflowDb } from '@/Interfaces'; -import { WorkflowExecutionService } from '@/workflows/workflowExecution.service'; -import type { WorkflowRunner } from '@/WorkflowRunner'; +import { WorkflowExecutionService } from '@/workflows/workflow-execution.service'; +import type { WorkflowRunner } from '@/workflow-runner'; const webhookNode: INode = { name: 'Webhook', diff --git a/packages/cli/src/workflows/workflowExecution.service.ts b/packages/cli/src/workflows/workflow-execution.service.ts similarity index 96% rename from packages/cli/src/workflows/workflowExecution.service.ts rename to packages/cli/src/workflows/workflow-execution.service.ts index 8ebe7aed0c..3611684ef1 100644 --- a/packages/cli/src/workflows/workflowExecution.service.ts +++ b/packages/cli/src/workflows/workflow-execution.service.ts @@ -19,7 +19,7 @@ import { import type { User } from '@db/entities/User'; import { ExecutionRepository } from '@db/repositories/execution.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; +import * as WorkflowHelpers from '@/workflow-helpers'; import type { WorkflowRequest } from '@/workflows/workflow.request'; import type { ExecutionPayload, @@ -27,11 +27,11 @@ import type { IWorkflowErrorData, IWorkflowExecutionDataProcess, } from '@/Interfaces'; -import { NodeTypes } from '@/NodeTypes'; -import { WorkflowRunner } from '@/WorkflowRunner'; -import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; -import { TestWebhooks } from '@/TestWebhooks'; -import { Logger } from '@/Logger'; +import { NodeTypes } from '@/node-types'; +import { WorkflowRunner } from '@/workflow-runner'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; +import { TestWebhooks } from '@/webhooks/test-webhooks'; +import { Logger } from '@/logger'; import type { Project } from '@/databases/entities/Project'; import { GlobalConfig } from '@n8n/config'; import { SubworkflowPolicyChecker } from '@/subworkflows/subworkflow-policy-checker.service'; diff --git a/packages/cli/test/unit/workflowHistoryHelper.test.ts b/packages/cli/src/workflows/workflow-history/__tests__/workflow-history-helper.ee.test.ts similarity index 91% rename from packages/cli/test/unit/workflowHistoryHelper.test.ts rename to packages/cli/src/workflows/workflow-history/__tests__/workflow-history-helper.ee.test.ts index 32a6cdd6f0..a4b7a70abf 100644 --- a/packages/cli/test/unit/workflowHistoryHelper.test.ts +++ b/packages/cli/src/workflows/workflow-history/__tests__/workflow-history-helper.ee.test.ts @@ -1,7 +1,7 @@ -import { License } from '@/License'; +import { License } from '@/license'; import config from '@/config'; -import { getWorkflowHistoryPruneTime } from '@/workflows/workflowHistory/workflowHistoryHelper.ee'; -import { mockInstance } from '../shared/mocking'; +import { getWorkflowHistoryPruneTime } from '@/workflows/workflow-history/workflow-history-helper.ee'; +import { mockInstance } from '@test/mocking'; let licensePruneTime = -1; diff --git a/packages/cli/test/unit/services/workflowHistory.service.ee.test.ts b/packages/cli/src/workflows/workflow-history/__tests__/workflow-history.service.ee.test.ts similarity index 91% rename from packages/cli/test/unit/services/workflowHistory.service.ee.test.ts rename to packages/cli/src/workflows/workflow-history/__tests__/workflow-history.service.ee.test.ts index 05ccae7005..0417908d1c 100644 --- a/packages/cli/test/unit/services/workflowHistory.service.ee.test.ts +++ b/packages/cli/src/workflows/workflow-history/__tests__/workflow-history.service.ee.test.ts @@ -2,10 +2,10 @@ import { mockClear } from 'jest-mock-extended'; import { User } from '@db/entities/User'; import { WorkflowHistoryRepository } from '@db/repositories/workflowHistory.repository'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; -import { WorkflowHistoryService } from '@/workflows/workflowHistory/workflowHistory.service.ee'; -import { Logger } from '@/Logger'; -import { mockInstance } from '../../shared/mocking'; -import { getWorkflow } from '../../integration/shared/workflow'; +import { WorkflowHistoryService } from '@/workflows/workflow-history/workflow-history.service.ee'; +import { Logger } from '@/logger'; +import { mockInstance } from '@test/mocking'; +import { getWorkflow } from '@test-integration/workflow'; const workflowHistoryRepository = mockInstance(WorkflowHistoryRepository); const logger = mockInstance(Logger); @@ -24,7 +24,7 @@ const testUser = Object.assign(new User(), { }); let isWorkflowHistoryEnabled = true; -jest.mock('@/workflows/workflowHistory/workflowHistoryHelper.ee', () => { +jest.mock('@/workflows/workflow-history/workflow-history-helper.ee', () => { return { isWorkflowHistoryEnabled: jest.fn(() => isWorkflowHistoryEnabled), }; diff --git a/packages/cli/src/workflows/workflowHistory/constants.ts b/packages/cli/src/workflows/workflow-history/constants.ts similarity index 100% rename from packages/cli/src/workflows/workflowHistory/constants.ts rename to packages/cli/src/workflows/workflow-history/constants.ts diff --git a/packages/cli/src/workflows/workflowHistory/workflowHistoryHelper.ee.ts b/packages/cli/src/workflows/workflow-history/workflow-history-helper.ee.ts similarity index 96% rename from packages/cli/src/workflows/workflowHistory/workflowHistoryHelper.ee.ts rename to packages/cli/src/workflows/workflow-history/workflow-history-helper.ee.ts index bf784c5aba..27da19b7da 100644 --- a/packages/cli/src/workflows/workflowHistory/workflowHistoryHelper.ee.ts +++ b/packages/cli/src/workflows/workflow-history/workflow-history-helper.ee.ts @@ -1,4 +1,4 @@ -import { License } from '@/License'; +import { License } from '@/license'; import config from '@/config'; import Container from 'typedi'; diff --git a/packages/cli/src/workflows/workflowHistory/workflowHistoryManager.ee.ts b/packages/cli/src/workflows/workflow-history/workflow-history-manager.ee.ts similarity index 90% rename from packages/cli/src/workflows/workflowHistory/workflowHistoryManager.ee.ts rename to packages/cli/src/workflows/workflow-history/workflow-history-manager.ee.ts index 4898666c88..d6eb214d8c 100644 --- a/packages/cli/src/workflows/workflowHistory/workflowHistoryManager.ee.ts +++ b/packages/cli/src/workflows/workflow-history/workflow-history-manager.ee.ts @@ -2,7 +2,10 @@ import { Service } from 'typedi'; import { DateTime } from 'luxon'; import { WorkflowHistoryRepository } from '@db/repositories/workflowHistory.repository'; import { WORKFLOW_HISTORY_PRUNE_INTERVAL } from './constants'; -import { getWorkflowHistoryPruneTime, isWorkflowHistoryEnabled } from './workflowHistoryHelper.ee'; +import { + getWorkflowHistoryPruneTime, + isWorkflowHistoryEnabled, +} from './workflow-history-helper.ee'; @Service() export class WorkflowHistoryManager { diff --git a/packages/cli/src/workflows/workflowHistory/workflowHistory.controller.ee.ts b/packages/cli/src/workflows/workflow-history/workflow-history.controller.ee.ts similarity index 94% rename from packages/cli/src/workflows/workflowHistory/workflowHistory.controller.ee.ts rename to packages/cli/src/workflows/workflow-history/workflow-history.controller.ee.ts index c57e3cb0b5..dd90037554 100644 --- a/packages/cli/src/workflows/workflowHistory/workflowHistory.controller.ee.ts +++ b/packages/cli/src/workflows/workflow-history/workflow-history.controller.ee.ts @@ -1,10 +1,10 @@ import { RestController, Get, Middleware } from '@/decorators'; import { WorkflowHistoryRequest } from '@/requests'; -import { WorkflowHistoryService } from './workflowHistory.service.ee'; +import { WorkflowHistoryService } from './workflow-history.service.ee'; import { Request, Response, NextFunction } from 'express'; -import { isWorkflowHistoryEnabled, isWorkflowHistoryLicensed } from './workflowHistoryHelper.ee'; +import { isWorkflowHistoryEnabled, isWorkflowHistoryLicensed } from './workflow-history-helper.ee'; -import { paginationListQueryMiddleware } from '@/middlewares/listQuery/pagination'; +import { paginationListQueryMiddleware } from '@/middlewares/list-query/pagination'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { SharedWorkflowNotFoundError } from '@/errors/shared-workflow-not-found.error'; import { WorkflowHistoryVersionNotFoundError } from '@/errors/workflow-history-version-not-found.error'; diff --git a/packages/cli/src/workflows/workflowHistory/workflowHistory.service.ee.ts b/packages/cli/src/workflows/workflow-history/workflow-history.service.ee.ts similarity index 96% rename from packages/cli/src/workflows/workflowHistory/workflowHistory.service.ee.ts rename to packages/cli/src/workflows/workflow-history/workflow-history.service.ee.ts index b92fc440cc..55cb085f42 100644 --- a/packages/cli/src/workflows/workflowHistory/workflowHistory.service.ee.ts +++ b/packages/cli/src/workflows/workflow-history/workflow-history.service.ee.ts @@ -4,8 +4,8 @@ import type { WorkflowHistory } from '@db/entities/WorkflowHistory'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { WorkflowHistoryRepository } from '@db/repositories/workflowHistory.repository'; import { Service } from 'typedi'; -import { isWorkflowHistoryEnabled } from './workflowHistoryHelper.ee'; -import { Logger } from '@/Logger'; +import { isWorkflowHistoryEnabled } from './workflow-history-helper.ee'; +import { Logger } from '@/logger'; import { SharedWorkflowNotFoundError } from '@/errors/shared-workflow-not-found.error'; import { WorkflowHistoryVersionNotFoundError } from '@/errors/workflow-history-version-not-found.error'; diff --git a/packages/cli/src/workflows/workflowSharing.service.ts b/packages/cli/src/workflows/workflow-sharing.service.ts similarity index 66% rename from packages/cli/src/workflows/workflowSharing.service.ts rename to packages/cli/src/workflows/workflow-sharing.service.ts index 93ef76438f..111e50b581 100644 --- a/packages/cli/src/workflows/workflowSharing.service.ts +++ b/packages/cli/src/workflows/workflow-sharing.service.ts @@ -8,12 +8,14 @@ import { RoleService } from '@/services/role.service'; import type { Scope } from '@n8n/permissions'; import type { ProjectRole } from '@/databases/entities/ProjectRelation'; import type { WorkflowSharingRole } from '@/databases/entities/SharedWorkflow'; +import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; @Service() export class WorkflowSharingService { constructor( private readonly sharedWorkflowRepository: SharedWorkflowRepository, private readonly roleService: RoleService, + private readonly projectRelationRepository: ProjectRelationRepository, ) {} /** @@ -27,11 +29,16 @@ export class WorkflowSharingService { async getSharedWorkflowIds( user: User, options: - | { scopes: Scope[] } - | { projectRoles: ProjectRole[]; workflowRoles: WorkflowSharingRole[] }, + | { scopes: Scope[]; projectId?: string } + | { projectRoles: ProjectRole[]; workflowRoles: WorkflowSharingRole[]; projectId?: string }, ): Promise { + const { projectId } = options; + if (user.hasGlobalScope('workflow:read')) { - const sharedWorkflows = await this.sharedWorkflowRepository.find({ select: ['workflowId'] }); + const sharedWorkflows = await this.sharedWorkflowRepository.find({ + select: ['workflowId'], + ...(projectId && { where: { projectId } }), + }); return sharedWorkflows.map(({ workflowId }) => workflowId); } @@ -59,4 +66,28 @@ export class WorkflowSharingService { return sharedWorkflows.map(({ workflowId }) => workflowId); } + + async getSharedWorkflowScopes( + workflowIds: string[], + user: User, + ): Promise> { + const projectRelations = await this.projectRelationRepository.findAllByUser(user.id); + const sharedWorkflows = + await this.sharedWorkflowRepository.getRelationsByWorkflowIdsAndProjectIds( + workflowIds, + projectRelations.map((p) => p.projectId), + ); + + return workflowIds.map((workflowId) => { + return [ + workflowId, + this.roleService.combineResourceScopes( + 'workflow', + user, + sharedWorkflows.filter((s) => s.workflowId === workflowId), + projectRelations, + ), + ]; + }); + } } diff --git a/packages/cli/src/workflows/workflowStaticData.service.ts b/packages/cli/src/workflows/workflow-static-data.service.ts similarity index 98% rename from packages/cli/src/workflows/workflowStaticData.service.ts rename to packages/cli/src/workflows/workflow-static-data.service.ts index 4c981acb87..d9c3564b26 100644 --- a/packages/cli/src/workflows/workflowStaticData.service.ts +++ b/packages/cli/src/workflows/workflow-static-data.service.ts @@ -1,7 +1,7 @@ import { Service } from 'typedi'; import { GlobalConfig } from '@n8n/config'; import { type IDataObject, type Workflow, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; import { isWorkflowIdValid } from '@/utils'; diff --git a/packages/cli/src/workflows/workflow.service.ee.ts b/packages/cli/src/workflows/workflow.service.ee.ts index 31029464f6..e3aae3a4e9 100644 --- a/packages/cli/src/workflows/workflow.service.ee.ts +++ b/packages/cli/src/workflows/workflow.service.ee.ts @@ -11,7 +11,7 @@ import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.reposi import { CredentialsService } from '@/credentials/credentials.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import { Logger } from '@/Logger'; +import { Logger } from '@/logger'; import type { WorkflowWithSharingsAndCredentials, WorkflowWithSharingsMetaDataAndCredentials, @@ -21,7 +21,7 @@ import { OwnershipService } from '@/services/ownership.service'; import { In, type EntityManager } from '@n8n/typeorm'; import { Project } from '@/databases/entities/Project'; import { ProjectService } from '@/services/project.service'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; import { TransferWorkflowError } from '@/errors/response-errors/transfer-workflow.error'; import { SharedWorkflow } from '@/databases/entities/SharedWorkflow'; diff --git a/packages/cli/src/workflows/workflow.service.ts b/packages/cli/src/workflows/workflow.service.ts index cf742acece..fb0619726f 100644 --- a/packages/cli/src/workflows/workflow.service.ts +++ b/packages/cli/src/workflows/workflow.service.ts @@ -1,4 +1,4 @@ -import Container, { Service } from 'typedi'; +import { Service } from 'typedi'; import { NodeApiError } from 'n8n-workflow'; import pick from 'lodash/pick'; import omit from 'lodash/omit'; @@ -11,21 +11,20 @@ import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { WorkflowTagMappingRepository } from '@db/repositories/workflowTagMapping.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { ActiveWorkflowManager } from '@/ActiveWorkflowManager'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; -import { validateEntity } from '@/GenericHelpers'; -import { ExternalHooks } from '@/ExternalHooks'; +import { ActiveWorkflowManager } from '@/active-workflow-manager'; +import * as WorkflowHelpers from '@/workflow-helpers'; +import { validateEntity } from '@/generic-helpers'; +import { ExternalHooks } from '@/external-hooks'; import { hasSharing, type ListQuery } from '@/requests'; import { TagService } from '@/services/tag.service'; -import { InternalHooks } from '@/InternalHooks'; import { OwnershipService } from '@/services/ownership.service'; -import { WorkflowHistoryService } from './workflowHistory/workflowHistory.service.ee'; -import { Logger } from '@/Logger'; +import { WorkflowHistoryService } from './workflow-history/workflow-history.service.ee'; +import { Logger } from '@/logger'; import { OrchestrationService } from '@/services/orchestration.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { RoleService } from '@/services/role.service'; -import { WorkflowSharingService } from './workflowSharing.service'; +import { WorkflowSharingService } from './workflow-sharing.service'; import { ProjectService } from '@/services/project.service'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import type { Scope } from '@n8n/permissions'; @@ -34,7 +33,7 @@ import type { EntityManager } from '@n8n/typeorm'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In } from '@n8n/typeorm'; import { SharedWorkflow } from '@/databases/entities/SharedWorkflow'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; @Service() export class WorkflowService { @@ -219,11 +218,10 @@ export class WorkflowService { } await this.externalHooks.run('workflow.afterUpdate', [updatedWorkflow]); - void Container.get(InternalHooks).onWorkflowSaved(user, updatedWorkflow, false); this.eventService.emit('workflow-saved', { user, - workflowId: updatedWorkflow.id, - workflowName: updatedWorkflow.name, + workflow: updatedWorkflow, + publicApi: false, }); if (updatedWorkflow.active) { @@ -282,8 +280,7 @@ export class WorkflowService { await this.workflowRepository.delete(workflowId); await this.binaryDataService.deleteMany(idsForDeletion); - void Container.get(InternalHooks).onWorkflowDeleted(user, workflowId, false); - this.eventService.emit('workflow-deleted', { user, workflowId }); + this.eventService.emit('workflow-deleted', { user, workflowId, publicApi: false }); await this.externalHooks.run('workflow.afterDelete', [workflowId]); return workflow; diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index d0ec019586..821bddd295 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -3,8 +3,8 @@ import { v4 as uuid } from 'uuid'; import axios from 'axios'; import * as Db from '@/Db'; -import * as ResponseHelper from '@/ResponseHelper'; -import * as WorkflowHelpers from '@/WorkflowHelpers'; +import * as ResponseHelper from '@/response-helper'; +import * as WorkflowHelpers from '@/workflow-helpers'; import type { IWorkflowResponse } from '@/Interfaces'; import config from '@/config'; import { Delete, Get, Patch, Post, ProjectScope, Put, RestController } from '@/decorators'; @@ -13,16 +13,15 @@ import { WorkflowEntity } from '@db/entities/WorkflowEntity'; import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository'; import { TagRepository } from '@db/repositories/tag.repository'; import { WorkflowRepository } from '@db/repositories/workflow.repository'; -import { validateEntity } from '@/GenericHelpers'; -import { ExternalHooks } from '@/ExternalHooks'; +import { validateEntity } from '@/generic-helpers'; +import { ExternalHooks } from '@/external-hooks'; import { WorkflowService } from './workflow.service'; -import { License } from '@/License'; -import { InternalHooks } from '@/InternalHooks'; +import { License } from '@/license'; import * as utils from '@/utils'; import { listQueryMiddleware } from '@/middlewares'; import { TagService } from '@/services/tag.service'; -import { WorkflowHistoryService } from './workflowHistory/workflowHistory.service.ee'; -import { Logger } from '@/Logger'; +import { WorkflowHistoryService } from './workflow-history/workflow-history.service.ee'; +import { Logger } from '@/logger'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; @@ -32,8 +31,8 @@ import { UserOnboardingService } from '@/services/userOnboarding.service'; import { CredentialsService } from '../credentials/credentials.service'; import { WorkflowRequest } from './workflow.request'; import { EnterpriseWorkflowService } from './workflow.service.ee'; -import { WorkflowExecutionService } from './workflowExecution.service'; -import { UserManagementMailer } from '@/UserManagement/email'; +import { WorkflowExecutionService } from './workflow-execution.service'; +import { UserManagementMailer } from '@/user-management/email'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { ProjectService } from '@/services/project.service'; import { ApplicationError } from 'n8n-workflow'; @@ -42,14 +41,13 @@ import { In, type FindOptionsRelations } from '@n8n/typeorm'; import type { Project } from '@/databases/entities/Project'; import { ProjectRelationRepository } from '@/databases/repositories/projectRelation.repository'; import { z } from 'zod'; -import { EventService } from '@/eventbus/event.service'; +import { EventService } from '@/events/event.service'; import { GlobalConfig } from '@n8n/config'; @RestController('/workflows') export class WorkflowsController { constructor( private readonly logger: Logger, - private readonly internalHooks: InternalHooks, private readonly externalHooks: ExternalHooks, private readonly tagRepository: TagRepository, private readonly enterpriseWorkflowService: EnterpriseWorkflowService, @@ -179,8 +177,13 @@ export class WorkflowsController { delete savedWorkflowWithMetaData.shared; await this.externalHooks.run('workflow.afterCreate', [savedWorkflow]); - void this.internalHooks.onWorkflowCreated(req.user, newWorkflow, project!, false); - this.eventService.emit('workflow-created', { user: req.user, workflow: newWorkflow }); + this.eventService.emit('workflow-created', { + user: req.user, + workflow: newWorkflow, + publicApi: false, + projectId: project!.id, + projectType: project!.type, + }); const scopes = await this.workflowService.getWorkflowScopes(req.user, savedWorkflow.id); @@ -454,7 +457,11 @@ export class WorkflowsController { newShareeIds = toShare; }); - void this.internalHooks.onWorkflowSharingUpdate(workflowId, req.user.id, shareWithIds); + this.eventService.emit('workflow-sharing-updated', { + workflowId, + userIdSharer: req.user.id, + userIdList: shareWithIds, + }); const projectsRelations = await this.projectRelationRepository.findBy({ projectId: In(newShareeIds), diff --git a/packages/cli/templates/form-trigger.handlebars b/packages/cli/templates/form-trigger.handlebars index a54168d2af..67818629f5 100644 --- a/packages/cli/templates/form-trigger.handlebars +++ b/packages/cli/templates/form-trigger.handlebars @@ -59,6 +59,9 @@ margin-bottom: 16px; } + .n8n-link { + padding-bottom: 24px; + } .n8n-link a { color: #7e8186; font-weight: 600; @@ -103,11 +106,12 @@ border-radius: 6px; width: 100%; font-size: 14px; - color: #909399; + color: #71747A; font-weight: 400; padding: 12px; } + form textarea:focus, form input:focus { outline: none; border-color: rgb(90, 76, 194); @@ -128,7 +132,7 @@ border-radius: 6px; width: 100%; font-size: 14px; - color: #909399; + color: #71747A; font-weight: 400; background-color: white; padding: 12px; @@ -141,6 +145,10 @@ sans-serif; } + ::placeholder { + opacity: 0.5; + } + #submit-btn { width: 100%; height: 48px; @@ -225,9 +233,80 @@ height: 18px; cursor: pointer; } - + /* required field ----------------------------- */ .form-required { } + label.form-required::after { + content: ' *'; + color: #ff6d5a; + } + + hr { + border: 0; + height: 1px; + border-top: 1px solid #dbdfe7; + margin-top: 24px; + margin-bottom: 24px; + display: none; + } + + .file-input-wrapper { + position: relative; + display: inline-block; + width: 100%; + } + input[type="file"] { + } + .clear-button { + position: absolute; + right: 5px; + top: 50%; + transform: translateY(-65%); + background-color: #7e8186; + border: none; + border-radius: 50%; + font-size: 14px; + font-weight: 600; + font-family: + Open Sans, + sans-serif; + color: white; + width: 20px; + height: 20px; + text-align: center; + line-height: 20px; + cursor: pointer; + display: none; + } + input[type="file"]:not(:empty) + .clear-button { + display: inline-block; + } + + @media only screen and (max-width: 500px) { + body { + background-color: white; + } + hr { + display: block; + } + .container { + width: 95%; + min-height: 100vh; + padding: 24px; + background-color: white; + border: 0px solid #dbdfe7; + border-radius: 0px; + box-shadow: 0px 0px 0px 0px white; + } + .card { + padding: 0px; + background-color: white; + border: 0px solid #dbdfe7; + border-radius: 0px; + box-shadow: 0px 0px 0px 0px white; + margin-bottom: 0px; + } + } @@ -238,20 +317,23 @@

This is test version of your form. Use it only for testing your Form Trigger.

+
{{/if}} + + {{#if validForm}}

{{formTitle}}

-

{{formDescription}}

+

{{formDescription}}

{{#each formFields}} {{#if isMultiSelect}}
- +
{{#each multiSelectOptions}}
@@ -268,7 +350,7 @@ {{#if isSelect}}
- +
+ placeholder="{{placeholder}}" + >{{defaultValue}} +

+ This field is required +

+
+ {{/if}} + + {{#if isFileInput}} +
+ + +

This field is required

@@ -299,12 +401,14 @@ {{#if isInput}}
- +

This field is required @@ -355,6 +459,7 @@

{{#if appendAttribution}} +
{{/if}} + + {{#if redirectUrl}} {{/if}} @@ -396,10 +503,13 @@
+ + + + diff --git a/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.stories.ts b/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.stories.ts new file mode 100644 index 0000000000..6d41905aed --- /dev/null +++ b/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.stories.ts @@ -0,0 +1,31 @@ +import AskAssistantButton from './AskAssistantButton.vue'; +import { action } from '@storybook/addon-actions'; +import type { StoryFn } from '@storybook/vue3'; + +export default { + title: 'Assistant/AskAssistantButton', + component: AskAssistantButton, + argTypes: {}, +}; + +const methods = { + onClick: action('click'), +}; + +const Template: StoryFn = (args, { argTypes }) => ({ + setup: () => ({ args }), + props: Object.keys(argTypes), + components: { + AskAssistantButton, + }, + template: + '
', + methods, +}); + +export const Button = Template.bind({}); + +export const Notifications = Template.bind({}); +Notifications.args = { + unreadCount: 1, +}; diff --git a/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.vue b/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.vue new file mode 100644 index 0000000000..5cb1824699 --- /dev/null +++ b/packages/design-system/src/components/AskAssistantButton/AskAssistantButton.vue @@ -0,0 +1,95 @@ + + + + + diff --git a/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.stories.ts b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.stories.ts new file mode 100644 index 0000000000..ec44622844 --- /dev/null +++ b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.stories.ts @@ -0,0 +1,240 @@ +import AskAssistantChat from './AskAssistantChat.vue'; +import type { StoryFn } from '@storybook/vue3'; +import type { ChatUI } from '../../types/assistant'; + +export default { + title: 'Assistant/AskAssistantChat', + component: AskAssistantChat, + argTypes: {}, +}; + +function getMessages(messages: ChatUI.AssistantMessage[]): ChatUI.AssistantMessage[] { + return messages; +} + +const methods = {}; + +const Template: StoryFn = (args, { argTypes }) => ({ + setup: () => ({ args }), + props: Object.keys(argTypes), + components: { + AskAssistantChat, + }, + template: '
', + methods, +}); + +export const DefaultPlaceholderChat = Template.bind({}); +DefaultPlaceholderChat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, +}; + +export const Chat = Template.bind({}); +Chat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '1', + type: 'text', + role: 'assistant', + content: 'Hi Max! Here is my top solution to fix the error in your **Transform data** node👇', + read: false, + }, + { + id: '1', + type: 'code-diff', + role: 'assistant', + description: 'Short solution description here that can spill over to two lines', + codeDiff: + '@@ -1,7 +1,6 @@\n-The Way that can be told of is not the eternal Way;\n-The name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\n-The Named is the mother of all things.\n+The named is the mother of all things.\n+\nTherefore let there always be non-being,\nso we may see their subtlety,\nAnd let there always be being,\n@@ -9,3 +8,6 @@\n The two are the same,\n But after they are produced,\n they have different names.\n+They both may be called deep and profound.\n+Deeper and more profound,\n+The door of all subtleties!', + suggestionId: 'test', + quickReplies: [ + { + type: 'new-suggestion', + text: 'Give me another solution', + }, + { + type: 'resolved', + text: 'All good', + }, + ], + read: false, + }, + { + id: '2', + type: 'text', + role: 'user', + content: 'Give it to me **ignore this markdown**', + read: false, + }, + { + id: '2', + type: 'block', + role: 'assistant', + title: 'Credential doesn’t have correct permissions to send a message', + content: + 'Solution steps:\n1. Lorem ipsum dolor sit amet, consectetur **adipiscing** elit. Proin id nulla placerat, tristique ex at, euismod dui.\n2. Copy this into somewhere\n3. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin id nulla placerat, tristique ex at, euismod dui.\n4. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin id nulla placerat, tristique ex at, euismod dui. \n Testing more code \n - Unordered item 1 \n - Unordered item 2', + read: false, + }, + { + id: '2', + type: 'code-diff', + role: 'assistant', + description: 'Short solution with min height', + codeDiff: + '@@ -1,7 +1,6 @@\n-The Way that can be told of is not the eternal Way;\n-The name that can be named is not the eternal name.\n+The door of all subtleties!', + quickReplies: [ + { + type: 'new-suggestion', + text: 'Give me another solution', + }, + { + type: 'resolved', + text: 'All good', + }, + ], + suggestionId: 'test', + read: false, + }, + ]), +}; + +export const JustSummary = Template.bind({}); +JustSummary.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + role: 'assistant', + type: 'block', + title: 'Credential doesn’t have correct permissions to send a message', + content: + 'Solution steps:\n1. Lorem ipsum dolor sit amet, consectetur **adipiscing** elit. Proin id nulla placerat, tristique ex at, euismod dui.\n2. Copy this into somewhere\n3. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin id nulla placerat, tristique ex at, euismod dui.\n4. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin id nulla placerat, tristique ex at, euismod dui. \n Testing more code \n - Unordered item 1 \n - Unordered item 2', + read: false, + }, + ]), +}; + +export const SummaryTitleStreaming = Template.bind({}); +SummaryTitleStreaming.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + role: 'assistant', + type: 'block', + title: 'Credential doesn’t have', + content: '', + read: false, + }, + ]), + streaming: true, +}; + +export const SummaryContentStreaming = Template.bind({}); +SummaryContentStreaming.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + role: 'assistant', + type: 'block', + title: 'Credential doesn’t have correct permissions to send a message', + content: 'Solution steps:\n1. Lorem ipsum dolor sit amet, consectetur', + read: false, + }, + ]), + streaming: true, +}; + +export const ErrorChat = Template.bind({}); +ErrorChat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + role: 'assistant', + type: 'error', + content: 'There was an error reaching the service', + read: false, + }, + ]), +}; + +export const EmptyStreamingChat = Template.bind({}); +EmptyStreamingChat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + type: 'text', + role: 'assistant', + content: '', + read: false, + }, + ]), + streaming: true, +}; + +export const StreamingChat = Template.bind({}); +StreamingChat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + type: 'text', + role: 'assistant', + content: 'I am thinking through this problem', + read: false, + }, + ]), + streaming: true, +}; + +export const EndOfSessionChat = Template.bind({}); +EndOfSessionChat.args = { + user: { + firstName: 'Max', + lastName: 'Test', + }, + messages: getMessages([ + { + id: '123', + type: 'text', + role: 'assistant', + content: "Great, glad I could help! I'm here whenever you need more help.", + read: false, + }, + { + id: '123', + role: 'assistant', + type: 'event', + eventName: 'end-session', + read: false, + }, + ]), +}; diff --git a/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue new file mode 100644 index 0000000000..a1ae2e8e88 --- /dev/null +++ b/packages/design-system/src/components/AskAssistantChat/AskAssistantChat.vue @@ -0,0 +1,491 @@ + + +