diff --git a/.github/workflows/benchmark-destroy-nightly.yml b/.github/workflows/benchmark-destroy-nightly.yml index 70d0d17685..0d87da7776 100644 --- a/.github/workflows/benchmark-destroy-nightly.yml +++ b/.github/workflows/benchmark-destroy-nightly.yml @@ -29,11 +29,14 @@ jobs: tenant-id: ${{ secrets.BENCHMARK_ARM_TENANT_ID }} subscription-id: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }} - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index 6c61d3bfc2..6ebf908062 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -48,11 +48,14 @@ jobs: with: terraform_version: '1.8.5' - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: pnpm + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/check-documentation-urls.yml b/.github/workflows/check-documentation-urls.yml index 0a2796abe7..ac7ca21335 100644 --- a/.github/workflows/check-documentation-urls.yml +++ b/.github/workflows/check-documentation-urls.yml @@ -16,11 +16,14 @@ jobs: steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml index e449891210..98037c520f 100644 --- a/.github/workflows/check-pr-title.yml +++ b/.github/workflows/check-pr-title.yml @@ -17,11 +17,14 @@ jobs: - name: Check out branch uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.yml index 4e77fbd580..f635550012 100644 --- a/.github/workflows/chromatic.yml +++ b/.github/workflows/chromatic.yml @@ -8,13 +8,14 @@ on: types: [submitted] concurrency: - group: chromatic-${{ github.event.pull_request.number || github.ref }} + group: chromatic-${{ github.event.pull_request.number || github.ref }}-${{github.event.review.state}} cancel-in-progress: true jobs: get-metadata: name: Get Metadata runs-on: ubuntu-latest + if: github.event.review.state == 'approved' steps: - name: Check out current commit uses: actions/checkout@v4 @@ -54,11 +55,16 @@ jobs: - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Publish to Chromatic diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml index b6972e1932..6c74f9ff6c 100644 --- a/.github/workflows/ci-master.yml +++ b/.github/workflows/ci-master.yml @@ -7,30 +7,35 @@ on: jobs: install-and-build: - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 + env: + NODE_OPTIONS: '--max-old-space-size=4096' timeout-minutes: 10 steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: useblacksmith/setup-node@v5 with: node-version: 20.x - cache: pnpm + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile - name: Setup build cache - uses: rharkor/caching-for-turbo@v1.5 + uses: useblacksmith/caching-for-turbo@v1 - name: Build run: pnpm build - name: Cache build artifacts - uses: actions/cache/save@v4.0.0 + uses: useblacksmith/cache/save@v5 with: path: ./packages/**/dist key: ${{ github.sha }}-base:build @@ -48,6 +53,7 @@ jobs: cacheKey: ${{ github.sha }}-base:build collectCoverage: ${{ matrix.node-version == '20.x' }} ignoreTurboCache: ${{ matrix.node-version == '20.x' }} + skipFrontendTests: ${{ matrix.node-version != '20.x' }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/ci-postgres-mysql.yml b/.github/workflows/ci-postgres-mysql.yml index f6e5e773cb..bd66409add 100644 --- a/.github/workflows/ci-postgres-mysql.yml +++ b/.github/workflows/ci-postgres-mysql.yml @@ -23,11 +23,16 @@ jobs: if: github.event_name != 'pull_request_review' || startsWith(github.event.pull_request.base.ref, 'release/') steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache @@ -52,11 +57,16 @@ jobs: DB_SQLITE_POOL_SIZE: 4 steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache @@ -81,11 +91,16 @@ jobs: DB_MYSQLDB_PASSWORD: password steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache @@ -118,11 +133,16 @@ jobs: DB_POSTGRESDB_POOL_SIZE: 1 # Detect connection pooling deadlocks steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index 1ce7427098..203b65a8c6 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -9,23 +9,28 @@ on: jobs: install-and-build: name: Install & Build - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 + env: + NODE_OPTIONS: '--max-old-space-size=4096' steps: - uses: actions/checkout@v4.1.1 with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: useblacksmith/setup-node@v5 with: node-version: 20.x - cache: pnpm + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile - name: Setup build cache - uses: rharkor/caching-for-turbo@v1.5 + uses: useblacksmith/caching-for-turbo@v1 - name: Build run: pnpm build @@ -37,7 +42,7 @@ jobs: run: pnpm typecheck - name: Cache build artifacts - uses: actions/cache/save@v4.0.0 + uses: useblacksmith/cache/save@v5 with: path: ./packages/**/dist key: ${{ github.sha }}-base:build diff --git a/.github/workflows/docker-base-image.yml b/.github/workflows/docker-base-image.yml index c32160e763..c2b5c9b6e6 100644 --- a/.github/workflows/docker-base-image.yml +++ b/.github/workflows/docker-base-image.yml @@ -20,26 +20,28 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.8.0 - name: Login to GitHub Container Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.11.0 + env: + DOCKER_BUILD_SUMMARY: false with: context: . file: ./docker/images/n8n-base/Dockerfile diff --git a/.github/workflows/docker-images-benchmark.yml b/.github/workflows/docker-images-benchmark.yml index b0aa6e997d..d4bf2f98a0 100644 --- a/.github/workflows/docker-images-benchmark.yml +++ b/.github/workflows/docker-images-benchmark.yml @@ -19,20 +19,22 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.8.0 - name: Login to GitHub Container Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.11.0 + env: + DOCKER_BUILD_SUMMARY: false with: context: . file: ./packages/@n8n/benchmark/Dockerfile diff --git a/.github/workflows/docker-images-custom.yml b/.github/workflows/docker-images-custom.yml new file mode 100644 index 0000000000..dfc9d77a03 --- /dev/null +++ b/.github/workflows/docker-images-custom.yml @@ -0,0 +1,83 @@ +name: Docker Custom Image CI +run-name: Build ${{ inputs.branch }} - ${{ inputs.user }} + +on: + workflow_dispatch: + inputs: + branch: + description: 'GitHub branch to create image off.' + required: true + tag: + description: 'Name of the docker tag to create.' + required: true + merge-master: + description: 'Merge with master.' + type: boolean + required: true + default: false + user: + description: '' + required: false + default: 'none' + start-url: + description: 'URL to call after workflow is kicked off.' + required: false + default: '' + success-url: + description: 'URL to call after Docker Image got built successfully.' + required: false + default: '' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Call Start URL - optionally + if: ${{ github.event.inputs.start-url != '' }} + run: curl -v -X POST -d 'url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' ${{github.event.inputs.start-url}} || echo "" + shell: bash + + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + ref: ${{ github.event.inputs.branch }} + + - name: Merge Master - optionally + if: github.event.inputs.merge-master + run: git remote add upstream https://github.com/n8n-io/n8n.git -f; git merge upstream/master --allow-unrelated-histories || echo "" + shell: bash + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.3.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.8.0 + + - name: Login to GHCR + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image to GHCR + uses: docker/build-push-action@v6.11.0 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: ./docker/images/n8n-custom/Dockerfile + build-args: | + N8N_RELEASE_TYPE=development + platforms: linux/amd64 + provenance: false + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ghcr.io/${{ github.repository_owner }}/n8n:${{ inputs.tag }} + + - name: Call Success URL - optionally + if: ${{ github.event.inputs.success-url != '' }} + run: curl -v ${{github.event.inputs.success-url}} || echo "" + shell: bash diff --git a/.github/workflows/docker-images-nightly.yml b/.github/workflows/docker-images-nightly.yml index b6831b6399..dbdde44078 100644 --- a/.github/workflows/docker-images-nightly.yml +++ b/.github/workflows/docker-images-nightly.yml @@ -1,74 +1,40 @@ name: Docker Nightly Image CI -run-name: Build ${{ inputs.branch }} - ${{ inputs.user }} on: schedule: - - cron: '0 1 * * *' + - cron: '0 0 * * *' workflow_dispatch: - inputs: - branch: - description: 'GitHub branch to create image off.' - required: true - default: 'master' - tag: - description: 'Name of the docker tag to create.' - required: true - default: 'nightly' - merge-master: - description: 'Merge with master.' - type: boolean - required: true - default: false - user: - description: '' - required: false - default: 'schedule' - start-url: - description: 'URL to call after workflow is kicked off.' - required: false - default: '' - success-url: - description: 'URL to call after Docker Image got built successfully.' - required: false - default: '' - -env: - N8N_TAG: ${{ inputs.tag || 'nightly' }} jobs: build: runs-on: ubuntu-latest - steps: - - name: Call Start URL - optionally - run: | - [[ "${{github.event.inputs.start-url}}" != "" ]] && curl -v -X POST -d 'url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' ${{github.event.inputs.start-url}} || echo "" - shell: bash - - name: Checkout uses: actions/checkout@v4.1.1 - with: - ref: ${{ github.event.inputs.branch || 'master' }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.8.0 + + - name: Login to GHCR + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Merge Master - optionally - run: | - [[ "${{github.event.inputs.merge-master}}" == "true" ]] && git remote add upstream https://github.com/n8n-io/n8n.git -f; git merge upstream/master --allow-unrelated-histories || echo "" - shell: bash - - - name: Build and push to DockerHub - uses: docker/build-push-action@v5.1.0 + - name: Build and push image to GHCR and DockerHub + uses: docker/build-push-action@v6.11.0 + env: + DOCKER_BUILD_SUMMARY: false with: context: . file: ./docker/images/n8n-custom/Dockerfile @@ -79,24 +45,6 @@ jobs: push: true cache-from: type=gha cache-to: type=gha,mode=max - tags: ${{ secrets.DOCKER_USERNAME }}/n8n:${{ env.N8N_TAG }} - - - name: Login to GitHub Container Registry - if: env.N8N_TAG == 'nightly' - uses: docker/login-action@v3.0.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push image to GHCR - if: env.N8N_TAG == 'nightly' - run: | - docker buildx imagetools create \ - --tag ghcr.io/${{ github.repository_owner }}/n8n:nightly \ + tags: | + ghcr.io/${{ github.repository_owner }}/n8n:nightly ${{ secrets.DOCKER_USERNAME }}/n8n:nightly - - - name: Call Success URL - optionally - run: | - [[ "${{github.event.inputs.success-url}}" != "" ]] && curl -v ${{github.event.inputs.success-url}} || echo "" - shell: bash diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml index b55d6728d2..8231896145 100644 --- a/.github/workflows/e2e-reusable.yml +++ b/.github/workflows/e2e-reusable.yml @@ -41,11 +41,6 @@ on: description: 'PR number to run tests for.' required: false type: number - node_view_version: - description: 'Node View version to run tests with.' - required: false - default: '1' - type: string secrets: CYPRESS_RECORD_KEY: description: 'Cypress record key.' @@ -165,7 +160,7 @@ jobs: spec: '${{ inputs.spec }}' env: NODE_OPTIONS: --dns-result-order=ipv4first - CYPRESS_NODE_VIEW_VERSION: ${{ inputs.node_view_version }} + CYPRESS_NODE_VIEW_VERSION: 2 CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} E2E_TESTS: true diff --git a/.github/workflows/e2e-tests-pr.yml b/.github/workflows/e2e-tests-pr.yml index 047ce2d13a..b244ea86ae 100644 --- a/.github/workflows/e2e-tests-pr.yml +++ b/.github/workflows/e2e-tests-pr.yml @@ -5,13 +5,14 @@ on: types: [submitted] concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} + group: e2e-${{ github.event.pull_request.number || github.ref }}-${{github.event.review.state}} cancel-in-progress: true jobs: get-metadata: name: Get Metadata runs-on: ubuntu-latest + if: github.event.review.state == 'approved' steps: - name: Check out current commit uses: actions/checkout@v4 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2f63f61bc6..e7400adecb 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -27,11 +27,6 @@ on: description: 'URL to call after workflow is done.' required: false default: '' - node_view_version: - description: 'Node View version to run tests with.' - required: false - default: '1' - type: string jobs: calls-start-url: @@ -51,7 +46,6 @@ jobs: branch: ${{ github.event.inputs.branch || 'master' }} user: ${{ github.event.inputs.user || 'PR User' }} spec: ${{ github.event.inputs.spec || 'e2e/*' }} - node_view_version: ${{ github.event.inputs.node_view_version || '1' }} secrets: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} diff --git a/.github/workflows/linting-reusable.yml b/.github/workflows/linting-reusable.yml index ed8d234940..a68cb4ebbb 100644 --- a/.github/workflows/linting-reusable.yml +++ b/.github/workflows/linting-reusable.yml @@ -17,23 +17,28 @@ on: jobs: lint: name: Lint - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 + env: + NODE_OPTIONS: '--max-old-space-size=4096' steps: - uses: actions/checkout@v4.1.1 with: ref: ${{ inputs.ref }} - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: useblacksmith/setup-node@v5 with: node-version: 20.x - cache: pnpm + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile - name: Setup build cache - uses: rharkor/caching-for-turbo@v1.5 + uses: useblacksmith/caching-for-turbo@v1 - name: Build if: ${{ inputs.cacheKey == '' }} @@ -41,10 +46,11 @@ jobs: - name: Restore cached build artifacts if: ${{ inputs.cacheKey != '' }} - uses: actions/cache/restore@v4.0.0 + uses: useblacksmith/cache/restore@v5 with: path: ./packages/**/dist key: ${{ inputs.cacheKey }} + fail-on-cache-miss: true - name: Lint Backend run: pnpm lint:backend diff --git a/.github/workflows/release-create-pr.yml b/.github/workflows/release-create-pr.yml index 03572e541c..dff2ce4bc8 100644 --- a/.github/workflows/release-create-pr.yml +++ b/.github/workflows/release-create-pr.yml @@ -35,13 +35,17 @@ jobs: fetch-depth: 0 ref: ${{ github.event.inputs.base-branch }} - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - run: npm install --prefix=.github/scripts --no-package-lock + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - name: Bump package versions run: | echo "NEXT_RELEASE=$(node .github/scripts/bump-versions.mjs)" >> $GITHUB_ENV diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 9196a7fccb..e8f8a44ee7 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -25,11 +25,15 @@ jobs: with: fetch-depth: 0 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Set release version in env @@ -73,26 +77,28 @@ jobs: fetch-depth: 0 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.8.0 - name: Login to GitHub Container Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.11.0 + env: + DOCKER_BUILD_SUMMARY: false with: context: ./docker/images/n8n build-args: | @@ -159,6 +165,14 @@ jobs: version: ${{ needs.publish-to-npm.outputs.release }} sourcemaps: packages/cli/dist packages/core/dist packages/nodes-base/dist packages/@n8n/n8n-nodes-langchain/dist + - name: Create a task runner release + uses: getsentry/action-release@v1.7.0 + continue-on-error: true + with: + projects: ${{ secrets.SENTRY_TASK_RUNNER_PROJECT }} + version: ${{ needs.publish-to-npm.outputs.release }} + sourcemaps: packages/core/dist packages/workflow/dist packages/@n8n/task-runner/dist + trigger-release-note: name: Trigger a release note needs: [publish-to-npm, create-github-release] diff --git a/.github/workflows/release-push-to-channel.yml b/.github/workflows/release-push-to-channel.yml index 3eda6d4ebb..74219e51c4 100644 --- a/.github/workflows/release-push-to-channel.yml +++ b/.github/workflows/release-push-to-channel.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/setup-node@v4.0.2 + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - run: | @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: docker/login-action@v3.0.0 + - uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: docker/login-action@v3.0.0 + - uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 2df829f7c6..661a0ffd9d 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -22,11 +22,16 @@ jobs: !contains(github.event.pull_request.labels.*.name, 'community') steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache @@ -48,11 +53,16 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4.1.1 - - run: corepack enable - - uses: actions/setup-node@v4.0.2 + + - uses: actions/setup-node@v4.2.0 with: node-version: 20.x - cache: 'pnpm' + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + - run: pnpm install --frozen-lockfile - name: Setup build cache diff --git a/.github/workflows/units-tests-dispatch.yml b/.github/workflows/units-tests-dispatch.yml index 5ad63f000e..72a9db5b6a 100644 --- a/.github/workflows/units-tests-dispatch.yml +++ b/.github/workflows/units-tests-dispatch.yml @@ -12,6 +12,11 @@ on: description: 'PR number to run tests for.' required: false type: number + skipFrontendTests: + description: 'Skip Frontend tests' + required: false + default: false + type: boolean jobs: prepare: @@ -37,3 +42,4 @@ jobs: uses: ./.github/workflows/units-tests-reusable.yml with: ref: ${{ needs.prepare.outputs.branch }} + skipFrontendTests: ${{ inputs.skipFrontendTests }} diff --git a/.github/workflows/units-tests-reusable.yml b/.github/workflows/units-tests-reusable.yml index 62eca74b15..fbb071900a 100644 --- a/.github/workflows/units-tests-reusable.yml +++ b/.github/workflows/units-tests-reusable.yml @@ -26,6 +26,10 @@ on: required: false default: false type: boolean + skipFrontendTests: + required: false + default: false + type: boolean secrets: CODECOV_TOKEN: description: 'Codecov upload token.' @@ -34,7 +38,7 @@ on: jobs: unit-test: name: Unit tests - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 env: TURBO_FORCE: ${{ inputs.ignoreTurboCache }} COVERAGE_ENABLED: ${{ inputs.collectCoverage }} @@ -43,18 +47,21 @@ jobs: with: ref: ${{ inputs.ref }} - - run: corepack enable - name: Use Node.js ${{ inputs.nodeVersion }} - uses: actions/setup-node@v4.0.2 + uses: useblacksmith/setup-node@v5 with: node-version: ${{ inputs.nodeVersion }} - cache: pnpm + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable - name: Install dependencies run: pnpm install --frozen-lockfile - name: Setup build cache - uses: rharkor/caching-for-turbo@v1.5 + uses: useblacksmith/caching-for-turbo@v1 - name: Build if: ${{ inputs.cacheKey == '' }} @@ -62,10 +69,11 @@ jobs: - name: Restore cached build artifacts if: ${{ inputs.cacheKey != '' }} - uses: actions/cache/restore@v4.0.0 + uses: useblacksmith/cache/restore@v5 with: path: ./packages/**/dist key: ${{ inputs.cacheKey }} + fail-on-cache-miss: true - name: Test Backend run: pnpm test:backend @@ -74,6 +82,7 @@ jobs: run: pnpm test:nodes - name: Test Frontend + if: ${{ !inputs.skipFrontendTests }} run: pnpm test:frontend - name: Upload coverage to Codecov diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b094f30ff..70c513f5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,199 @@ -## [1.74.1](https://github.com/n8n-io/n8n/compare/n8n@1.74.0...n8n@1.74.1) (2025-01-09) +# [1.78.0](https://github.com/n8n-io/n8n/compare/n8n@1.77.0...n8n@1.78.0) (2025-02-06) ### Bug Fixes -* **editor:** Fix parameter input validation ([#12532](https://github.com/n8n-io/n8n/issues/12532)) ([6f757f1](https://github.com/n8n-io/n8n/commit/6f757f10bd9102394d2a0b6bbc795f90444f66d2)) +* **AI Agent Node:** Ignore SSL errors option for SQLAgent ([#13052](https://github.com/n8n-io/n8n/issues/13052)) ([a90529f](https://github.com/n8n-io/n8n/commit/a90529fd51ca88bc9640d24490dbeb2023c98e30)) +* **Code Node:** Do not validate code within comments ([#12938](https://github.com/n8n-io/n8n/issues/12938)) ([cdfa225](https://github.com/n8n-io/n8n/commit/cdfa22593b69cf647c2a798d6571a9bbbd11c1b2)) +* **core:** "Respond to Webhook" should work with workflows with waiting nodes ([#12806](https://github.com/n8n-io/n8n/issues/12806)) ([e8635f2](https://github.com/n8n-io/n8n/commit/e8635f257433748f4d7d2c4b0ae794de6bff5b28)) +* **core:** Do not emit `workflow-post-execute` event for waiting executions ([#13065](https://github.com/n8n-io/n8n/issues/13065)) ([1593b6c](https://github.com/n8n-io/n8n/commit/1593b6cb4112ab2a85ca93c4eaec7d5f088895b1)) +* **core:** Do not enable strict type validation by default for resource mapper ([#13037](https://github.com/n8n-io/n8n/issues/13037)) ([fdcff90](https://github.com/n8n-io/n8n/commit/fdcff9082b97314f8b04579ab6fa81c724916320)) +* **core:** Fix empty node execution stack ([#12945](https://github.com/n8n-io/n8n/issues/12945)) ([7031569](https://github.com/n8n-io/n8n/commit/7031569a028bcc85558fcb614f8143d68a7f81f0)) +* **core:** Only use new resource mapper type validation when it is enabled ([#13099](https://github.com/n8n-io/n8n/issues/13099)) ([a37c8e8](https://github.com/n8n-io/n8n/commit/a37c8e8fb86aaa3244ac13500ffa0e7c0d809a6f)) +* **editor:** Actually enforce the version and don't break for old values in local storage ([#13025](https://github.com/n8n-io/n8n/issues/13025)) ([884a7e2](https://github.com/n8n-io/n8n/commit/884a7e23f84258756d8dcdd2dfe933bdedf61adc)) +* **editor:** Add telemetry to source control feature ([#13016](https://github.com/n8n-io/n8n/issues/13016)) ([18eaa54](https://github.com/n8n-io/n8n/commit/18eaa5423dfc9348374c2cff4ae0e6f152268fbb)) +* **editor:** Allow switch to `Fixed` for boolean and number parameters with invalid expressions ([#12948](https://github.com/n8n-io/n8n/issues/12948)) ([118be24](https://github.com/n8n-io/n8n/commit/118be24d25f001525ced03d9426a6129fa5a2053)) +* **editor:** Allow to re-open sub-connection node creator if already active ([#13041](https://github.com/n8n-io/n8n/issues/13041)) ([16d59e9](https://github.com/n8n-io/n8n/commit/16d59e98edc427bf68edbce4cd2174a44d6dcfb1)) +* **editor:** Code node overwrites code when switching nodes after edits ([#13078](https://github.com/n8n-io/n8n/issues/13078)) ([00e3ebc](https://github.com/n8n-io/n8n/commit/00e3ebc9e2e0b8cc2d88b678c3a2a21602dac010)) +* **editor:** Fix execution running status listener for chat messages ([#12951](https://github.com/n8n-io/n8n/issues/12951)) ([4d55a29](https://github.com/n8n-io/n8n/commit/4d55a294600dc2c86f6f7019da923b66a4b9de7e)) +* **editor:** Fix position of connector buttons when the line is straight ([#13034](https://github.com/n8n-io/n8n/issues/13034)) ([3a908ac](https://github.com/n8n-io/n8n/commit/3a908aca17f0bc1cf5fb5eb8813cc94f27f0bcdf)) +* **editor:** Fix showing and hiding canvas edge toolbar when hovering ([#13009](https://github.com/n8n-io/n8n/issues/13009)) ([ac7bc4f](https://github.com/n8n-io/n8n/commit/ac7bc4f1911f913233eeeae5d229432fdff332c4)) +* **editor:** Make AI transform node read only in executions view ([#12970](https://github.com/n8n-io/n8n/issues/12970)) ([ce1deb8](https://github.com/n8n-io/n8n/commit/ce1deb8aea528eef996fc774d0fff1dc61df5843)) +* **editor:** Prevent infinite loop in expressions crashing the browser ([#12732](https://github.com/n8n-io/n8n/issues/12732)) ([8c2dbcf](https://github.com/n8n-io/n8n/commit/8c2dbcfeced70a0a84137773269cc6db2928d174)) +* **editor:** Refine push modal layout ([#12886](https://github.com/n8n-io/n8n/issues/12886)) ([212a5bf](https://github.com/n8n-io/n8n/commit/212a5bf23eb11cc3296e7a8d002a4b7727d5193c)) +* **editor:** SchemaView renders duplicate structures properly ([#12943](https://github.com/n8n-io/n8n/issues/12943)) ([0d8a544](https://github.com/n8n-io/n8n/commit/0d8a544975f72724db931778d7e3ace8a12b6cfc)) +* **editor:** Update node issues when opening execution ([#12972](https://github.com/n8n-io/n8n/issues/12972)) ([1a91523](https://github.com/n8n-io/n8n/commit/1a915239c6571d7744023c6df6242dabe97c912e)) +* **editor:** Use correct connection index when connecting adjancent nodes after deleting a node ([#12973](https://github.com/n8n-io/n8n/issues/12973)) ([c7a15d5](https://github.com/n8n-io/n8n/commit/c7a15d5980d181a865f8e2ec6a5f70d0681dcf56)) +* **GitHub Node:** Don't truncate filenames retrieved from GitHub ([#12923](https://github.com/n8n-io/n8n/issues/12923)) ([7e18447](https://github.com/n8n-io/n8n/commit/7e1844757fe0d544e8881d229d16af95ed53fb21)) +* **Google Cloud Firestore Node:** Fix potential prototype pollution vulnerability ([#13035](https://github.com/n8n-io/n8n/issues/13035)) ([f150f79](https://github.com/n8n-io/n8n/commit/f150f79ad6c7d43e036688b1de8d6c2c8140aca9)) +* Increment runIndex in WorkflowToolV2 tool executions to avoid reusing out of date inputs ([#13008](https://github.com/n8n-io/n8n/issues/13008)) ([cc907fb](https://github.com/n8n-io/n8n/commit/cc907fbca9aa00fe07dd54a2fcac8983f2321ad1)) +* Sync partial execution version of FE and BE, also allow enforcing a specific version ([#12840](https://github.com/n8n-io/n8n/issues/12840)) ([a155043](https://github.com/n8n-io/n8n/commit/a15504329bac582225185705566297d9cc27bf73)) +* **Wise Node:** Use ISO formatting for timestamps ([#10288](https://github.com/n8n-io/n8n/issues/10288)) ([1a2d39a](https://github.com/n8n-io/n8n/commit/1a2d39a158c9a61bdaf11124b09ae70de65ebbf1)) + + +### Features + +* Add reusable frontend `composables` package ([#13077](https://github.com/n8n-io/n8n/issues/13077)) ([ef87da4](https://github.com/n8n-io/n8n/commit/ef87da4c193a08e089e48044906a4f5ce9959a22)) +* Add support for client credentials with Azure Log monitor ([#13038](https://github.com/n8n-io/n8n/issues/13038)) ([2c2d631](https://github.com/n8n-io/n8n/commit/2c2d63157b7866f1a68cc45c5823e29570ccff77)) +* Allow multi API creation via the UI ([#12845](https://github.com/n8n-io/n8n/issues/12845)) ([ad3250c](https://github.com/n8n-io/n8n/commit/ad3250ceb0df84379917e684d54d4100e3bf44f5)) +* Allow setting API keys expiration ([#12954](https://github.com/n8n-io/n8n/issues/12954)) ([9bcbc2c](https://github.com/n8n-io/n8n/commit/9bcbc2c2ccbb88537e9b7554c92b631118d870f1)) +* **core:** Add sorting to GET `/workflows` endpoint ([#13029](https://github.com/n8n-io/n8n/issues/13029)) ([b60011a](https://github.com/n8n-io/n8n/commit/b60011a1808d47f32ab84e685dba0e915e82df8f)) +* **core:** Enable usage as a tool for more nodes ([#12930](https://github.com/n8n-io/n8n/issues/12930)) ([9deb759](https://github.com/n8n-io/n8n/commit/9deb75916e4eb63b899ba79b40cbd24b69a752db)) +* **core:** Handle Declarative nodes more like regular nodes ([#13007](https://github.com/n8n-io/n8n/issues/13007)) ([a65a9e6](https://github.com/n8n-io/n8n/commit/a65a9e631b13bbe70ad64727fb1109ae7cd014eb)) +* **Discord Node:** New sendAndWait operation ([#12894](https://github.com/n8n-io/n8n/issues/12894)) ([d47bfdd](https://github.com/n8n-io/n8n/commit/d47bfddd656367454b51da39cf87dbfb2bd59eb2)) +* **editor:** Display schema preview for unexecuted nodes ([#12901](https://github.com/n8n-io/n8n/issues/12901)) ([0063bbb](https://github.com/n8n-io/n8n/commit/0063bbb30b45b3af92aff4c0f76b905d50a71a2d)) +* **editor:** Easy $fromAI Button for AI Tools ([#12587](https://github.com/n8n-io/n8n/issues/12587)) ([2177376](https://github.com/n8n-io/n8n/commit/21773764d37c37a6464a3885d3fa548a5feb4fd8)) +* **editor:** Show fixed collection parameter issues in UI ([#12899](https://github.com/n8n-io/n8n/issues/12899)) ([12d686c](https://github.com/n8n-io/n8n/commit/12d686ce52694f4c0b88f92a744451c1b0c66dec)) +* **Facebook Graph API Node:** Update node to support API v22.0 ([#13024](https://github.com/n8n-io/n8n/issues/13024)) ([0bc0fc6](https://github.com/n8n-io/n8n/commit/0bc0fc6c1226688c29bf5f8f0ba7e8f244e16fbc)) +* **HTTP Request Tool Node:** Relax binary data detection ([#13048](https://github.com/n8n-io/n8n/issues/13048)) ([b67a003](https://github.com/n8n-io/n8n/commit/b67a003e0b154d4e8c04392bec1c7b28171b5908)) +* Human in the loop section ([#12883](https://github.com/n8n-io/n8n/issues/12883)) ([9590e5d](https://github.com/n8n-io/n8n/commit/9590e5d58b8964de9ce901bf07b537926d18b6b7)) +* **n8n Form Node:** Add Hidden Fields ([#12803](https://github.com/n8n-io/n8n/issues/12803)) ([0da1114](https://github.com/n8n-io/n8n/commit/0da1114981978e371b216bdabc0c3bbdceeefa09)) +* **n8n Form Node:** Respond with Text ([#12979](https://github.com/n8n-io/n8n/issues/12979)) ([182fc15](https://github.com/n8n-io/n8n/commit/182fc150bec62e9a5e2801d6c403e4a6bd35f728)) +* **OpenAI Chat Model Node, OpenAI Node:** Include o3 models in model selection ([#13005](https://github.com/n8n-io/n8n/issues/13005)) ([37d152c](https://github.com/n8n-io/n8n/commit/37d152c148cafbe493c22e07f5d55ff24fcb0ca4)) +* **Summarize Node:** Preserves original field data type ([#13069](https://github.com/n8n-io/n8n/issues/13069)) ([be5e49d](https://github.com/n8n-io/n8n/commit/be5e49d56c09d65c9768e948471626cfd3606c0c)) + + + +# [1.77.0](https://github.com/n8n-io/n8n/compare/n8n@1.76.0...n8n@1.77.0) (2025-01-29) + + +### Bug Fixes + +* **core:** Account for pre-execution failure in scaling mode ([#12815](https://github.com/n8n-io/n8n/issues/12815)) ([b4d27c4](https://github.com/n8n-io/n8n/commit/b4d27c49e32bfacbd2690bf1c07194562f6a4a61)) +* **core:** Display the last activated plan name when multiple are activated ([#12835](https://github.com/n8n-io/n8n/issues/12835)) ([03365f0](https://github.com/n8n-io/n8n/commit/03365f096d3d5c8e3a6537f37cda412959705346)) +* **core:** Fix possible corruption of OAuth2 credential ([#12880](https://github.com/n8n-io/n8n/issues/12880)) ([ac84ea1](https://github.com/n8n-io/n8n/commit/ac84ea14452cbcec95f14073e8e70427169e6a7f)) +* **core:** Fix usage of external libs in task runner ([#12788](https://github.com/n8n-io/n8n/issues/12788)) ([3d9d5bf](https://github.com/n8n-io/n8n/commit/3d9d5bf9d58f3c49830d42a140d6c8c6b59952dc)) +* **core:** Handle max stalled count error better ([#12824](https://github.com/n8n-io/n8n/issues/12824)) ([eabf160](https://github.com/n8n-io/n8n/commit/eabf1609577cd94a6bad5020c34378d840a13bc0)) +* **core:** Improve error handling in credential decryption and parsing ([#12868](https://github.com/n8n-io/n8n/issues/12868)) ([0c86bf2](https://github.com/n8n-io/n8n/commit/0c86bf2b3761bb93fd3cedba7a483ae5d97bd332)) +* **core:** Renew license on startup for instances with detached floating entitlements ([#12884](https://github.com/n8n-io/n8n/issues/12884)) ([f32eef8](https://github.com/n8n-io/n8n/commit/f32eef85bd066ee9b54d110355c6b80124d67437)) +* **core:** Update execution entity and execution data in transaction ([#12756](https://github.com/n8n-io/n8n/issues/12756)) ([1f43181](https://github.com/n8n-io/n8n/commit/1f4318136011bffaad04527790a9eba79effce35)) +* **core:** Validate credential data before encryption ([#12885](https://github.com/n8n-io/n8n/issues/12885)) ([3d27a14](https://github.com/n8n-io/n8n/commit/3d27a1498702206b738cf978d037191306cec42b)) +* **editor:** Add notice when user hits the limit for execution metadata item length ([#12676](https://github.com/n8n-io/n8n/issues/12676)) ([02df25c](https://github.com/n8n-io/n8n/commit/02df25c450a0a384a32d0815d8a2faec7562a8ae)) +* **editor:** Don't send run data for full manual executions ([#12687](https://github.com/n8n-io/n8n/issues/12687)) ([9139dc3](https://github.com/n8n-io/n8n/commit/9139dc3c2916186648fb5bf63d14fcb90773eb1c)) +* **editor:** Fix sub-execution links in empty output tables ([#12781](https://github.com/n8n-io/n8n/issues/12781)) ([114ed88](https://github.com/n8n-io/n8n/commit/114ed88368d137443b9c6605d4fe11b02053549d)) +* **editor:** Fix workflow move project select filtering ([#12764](https://github.com/n8n-io/n8n/issues/12764)) ([358d284](https://github.com/n8n-io/n8n/commit/358d2843e5e468071d6764419169811e93138c35)) +* **editor:** Focus executions iframe when n8n is ready to delegate keyboard events ([#12741](https://github.com/n8n-io/n8n/issues/12741)) ([d506218](https://github.com/n8n-io/n8n/commit/d5062189dbca02dfdf485fc220cc2a7b05e3e6cc)) +* **editor:** Handle large payloads in the AI Assistant requests better ([#12747](https://github.com/n8n-io/n8n/issues/12747)) ([eb4dea1](https://github.com/n8n-io/n8n/commit/eb4dea1ca891bb7ac07c8bbbae8803de080c4623)) +* **editor:** Hide Set up Template button for empty workflows ([#12808](https://github.com/n8n-io/n8n/issues/12808)) ([36e615b](https://github.com/n8n-io/n8n/commit/36e615b28f395623457bbb9bf4ab6fd69102b6ea)) +* **editor:** Load appropriate credentials in canvas V2 for new workflow ([#12722](https://github.com/n8n-io/n8n/issues/12722)) ([2020dc5](https://github.com/n8n-io/n8n/commit/2020dc502feae6cae827dfbcc40ffed89bcc334a)) +* **editor:** Properly set active project in new canvas ([#12810](https://github.com/n8n-io/n8n/issues/12810)) ([648c6f9](https://github.com/n8n-io/n8n/commit/648c6f9315b16b885e04716e7e0035a73b358fb0)) +* **editor:** Render inline SVGs correctly on the external secrets settings page ([#12802](https://github.com/n8n-io/n8n/issues/12802)) ([5820ade](https://github.com/n8n-io/n8n/commit/5820ade1e4b9d638c9b6369aef369d6dc9320da6)) +* **editor:** Show input selector when node has error ([#12813](https://github.com/n8n-io/n8n/issues/12813)) ([5b760e7](https://github.com/n8n-io/n8n/commit/5b760e7f7fc612b10307b4871e24b549f5d9d420)) +* **editor:** Show mappings by default in sub-node NDVs when the root node isn't executed ([#12642](https://github.com/n8n-io/n8n/issues/12642)) ([fb662dd](https://github.com/n8n-io/n8n/commit/fb662dd95cae3bc51d05d05e32e772d05adafa1e)) +* **Postgres PGVector Store Node:** Release postgres connections back to the pool ([#12723](https://github.com/n8n-io/n8n/issues/12723)) ([663dfb4](https://github.com/n8n-io/n8n/commit/663dfb48defd944f88f0ecc4f3347ea4f8a7c831)) + + +### Features + +* Add DeepSeek Chat Model node ([#12873](https://github.com/n8n-io/n8n/issues/12873)) ([9918afa](https://github.com/n8n-io/n8n/commit/9918afa51b16116abb73692a66df84e48128f406)) +* Add OpenRouter node ([#12882](https://github.com/n8n-io/n8n/issues/12882)) ([dc85b02](https://github.com/n8n-io/n8n/commit/dc85b022d111d1e8b038ca1a9f6a1041f19cf2b0)) +* Add timeout options to sendAndWait operations ([#12753](https://github.com/n8n-io/n8n/issues/12753)) ([3e9f24d](https://github.com/n8n-io/n8n/commit/3e9f24ddf462349145d89fe183313c95512c699b)) +* **API:** Add route for schema static files ([#12770](https://github.com/n8n-io/n8n/issues/12770)) ([d981b56](https://github.com/n8n-io/n8n/commit/d981b5659a26f92b11e5d0cd5570504fd683626c)) +* **core:** Explicitly report external hook failures ([#12830](https://github.com/n8n-io/n8n/issues/12830)) ([a24e442](https://github.com/n8n-io/n8n/commit/a24e4420bb9023f808acd756d125dffaea325968)) +* **core:** Rename two task runner env vars ([#12763](https://github.com/n8n-io/n8n/issues/12763)) ([60187ca](https://github.com/n8n-io/n8n/commit/60187cab9bc9d21aa6ba710d772c068324e429f1)) +* **editor:** Add evaluation workflow and enhance workflow selector with pinned data support ([#12773](https://github.com/n8n-io/n8n/issues/12773)) ([be967eb](https://github.com/n8n-io/n8n/commit/be967ebec07fab223513f93f50bcc389b9a4c548)) +* **editor:** Always keep at least one executing node indicator in the workflow ([#12829](https://github.com/n8n-io/n8n/issues/12829)) ([c25c613](https://github.com/n8n-io/n8n/commit/c25c613a04a6773fa4014d9a0d290e443bcabbe0)) +* **Google Chat Node:** Updates ([#12827](https://github.com/n8n-io/n8n/issues/12827)) ([e146ad0](https://github.com/n8n-io/n8n/commit/e146ad021a0be22cf51bafa3c015d03550e03d97)) +* **Microsoft Outlook Node:** New operation sendAndWait ([#12795](https://github.com/n8n-io/n8n/issues/12795)) ([f4bf55f](https://github.com/n8n-io/n8n/commit/f4bf55f0d8278ff954344cf6397c10d8261b39a4)) +* **n8n Form Node:** Add read-only/custom HTML form elements ([#12760](https://github.com/n8n-io/n8n/issues/12760)) ([ba8aa39](https://github.com/n8n-io/n8n/commit/ba8aa3921613c590caaac627fbb9837ccaf87783)) +* **Send Email Node:** New operation sendAndWait ([#12775](https://github.com/n8n-io/n8n/issues/12775)) ([a197fbb](https://github.com/n8n-io/n8n/commit/a197fbb21b5642843d8bc3e657049aca99e0729d)) +* **Summarize Node:** Turns error when field not found in items into warning ([#11889](https://github.com/n8n-io/n8n/issues/11889)) ([d7dda3f](https://github.com/n8n-io/n8n/commit/d7dda3f5de52925e554455f9f10e51bd173ea856)) +* **Telegram Node:** New operation sendAndWait ([#12771](https://github.com/n8n-io/n8n/issues/12771)) ([2c58d47](https://github.com/n8n-io/n8n/commit/2c58d47f8eee1f865ecc1eeb89aa20c69c28abae)) + + + +# [1.76.0](https://github.com/n8n-io/n8n/compare/n8n@1.75.0...n8n@1.76.0) (2025-01-22) + + +### Bug Fixes + +* **core:** Align saving behavior in `workflowExecuteAfter` hooks ([#12731](https://github.com/n8n-io/n8n/issues/12731)) ([9d76210](https://github.com/n8n-io/n8n/commit/9d76210a570e025d01d1f6596667abf40fbd8d12)) +* **core:** AugmentObject should handle the constructor property correctly ([#12744](https://github.com/n8n-io/n8n/issues/12744)) ([36bc164](https://github.com/n8n-io/n8n/commit/36bc164da486f2e2d05091b457b8eea6521ca22e)) +* **core:** Fix keyboard shortcuts for non-ansi layouts ([#12672](https://github.com/n8n-io/n8n/issues/12672)) ([4c8193f](https://github.com/n8n-io/n8n/commit/4c8193fedc2e3967c9a06c0652483128df509653)) +* **core:** Fix license CLI commands showing incorrect renewal setting ([#12759](https://github.com/n8n-io/n8n/issues/12759)) ([024ada8](https://github.com/n8n-io/n8n/commit/024ada822c1bc40958e594bb08707cf77d3397ec)) +* **core:** Fix license initialization failure on startup ([#12737](https://github.com/n8n-io/n8n/issues/12737)) ([ac2f647](https://github.com/n8n-io/n8n/commit/ac2f6476c114f51fafb9b7b66e41e0c87f4a1bf6)) +* **core:** Recover successful data-less executions ([#12720](https://github.com/n8n-io/n8n/issues/12720)) ([a39b8bd](https://github.com/n8n-io/n8n/commit/a39b8bd32be50c8323e415f820b25b4bcb81d960)) +* **core:** Remove run data of utility nodes for partial executions v2 ([#12673](https://github.com/n8n-io/n8n/issues/12673)) ([b66a9dc](https://github.com/n8n-io/n8n/commit/b66a9dc8fb6f7b19122cacbb7e2f86b4c921c3fb)) +* **core:** Sync `hookFunctionsSave` and `hookFunctionsSaveWorker` ([#12740](https://github.com/n8n-io/n8n/issues/12740)) ([d410b8f](https://github.com/n8n-io/n8n/commit/d410b8f5a7e99658e1e8dcb2e02901bd01ce9c59)) +* **core:** Update isDocker check to return true on kubernetes/containerd ([#12603](https://github.com/n8n-io/n8n/issues/12603)) ([c55dac6](https://github.com/n8n-io/n8n/commit/c55dac66ed97a2317d4c696c3b505790ec5d72fe)) +* **editor:** Add unicode code points to expression language for emoji ([#12633](https://github.com/n8n-io/n8n/issues/12633)) ([819ebd0](https://github.com/n8n-io/n8n/commit/819ebd058d1d60b3663d92b4a652728da7134a3b)) +* **editor:** Correct missing whitespace in JSON output ([#12677](https://github.com/n8n-io/n8n/issues/12677)) ([b098b19](https://github.com/n8n-io/n8n/commit/b098b19c7f0e3a9848c3fcfa012999050f2d3c7a)) +* **editor:** Defer crypto.randomUUID call in CodeNodeEditor ([#12630](https://github.com/n8n-io/n8n/issues/12630)) ([58f6532](https://github.com/n8n-io/n8n/commit/58f6532630bacd288d3c0a79b40150f465898419)) +* **editor:** Fix Code node bug erasing and overwriting code when switching between nodes ([#12637](https://github.com/n8n-io/n8n/issues/12637)) ([02d953d](https://github.com/n8n-io/n8n/commit/02d953db34ec4e44977a8ca908628b62cca82fde)) +* **editor:** Fix execution list hover & selection colour in dark mode ([#12628](https://github.com/n8n-io/n8n/issues/12628)) ([95c40c0](https://github.com/n8n-io/n8n/commit/95c40c02cb8fef77cf633cf5aec08e98746cff36)) +* **editor:** Fix JsonEditor with expressions ([#12739](https://github.com/n8n-io/n8n/issues/12739)) ([56c93ca](https://github.com/n8n-io/n8n/commit/56c93caae026738c1c0bebb4187b238e34a330f6)) +* **editor:** Fix navbar height flickering during load ([#12738](https://github.com/n8n-io/n8n/issues/12738)) ([a96b3f0](https://github.com/n8n-io/n8n/commit/a96b3f0091798a52bb33107b919b5d8287ba7506)) +* **editor:** Open chat when executing agent node in canvas v2 ([#12617](https://github.com/n8n-io/n8n/issues/12617)) ([457edd9](https://github.com/n8n-io/n8n/commit/457edd99bb853d8ccf3014605d5823933f3c0bc6)) +* **editor:** Partial execution of a workflow with manual chat trigger ([#12662](https://github.com/n8n-io/n8n/issues/12662)) ([2f81b29](https://github.com/n8n-io/n8n/commit/2f81b29d341535b512df0aa01b25a91d109f113f)) +* **editor:** Show connector label above the line when it's straight ([#12622](https://github.com/n8n-io/n8n/issues/12622)) ([c97bd48](https://github.com/n8n-io/n8n/commit/c97bd48a77643b9c2a5d7218e21b957af15cee0b)) +* **editor:** Show run workflow button when chat trigger has pinned data ([#12616](https://github.com/n8n-io/n8n/issues/12616)) ([da8aafc](https://github.com/n8n-io/n8n/commit/da8aafc0e3a1b5d862f0723d0d53d2c38bcaebc3)) +* **editor:** Update workflow re-initialization to use query parameter ([#12650](https://github.com/n8n-io/n8n/issues/12650)) ([982131a](https://github.com/n8n-io/n8n/commit/982131a75a32f741c120156826c303989aac189c)) +* **Execute Workflow Node:** Pass binary data to sub-workflow ([#12635](https://github.com/n8n-io/n8n/issues/12635)) ([e9c152e](https://github.com/n8n-io/n8n/commit/e9c152e369a4c2762bd8e6ad17eaa704bb3771bb)) +* **Google Gemini Chat Model Node:** Add base URL support for Google Gemini Chat API ([#12643](https://github.com/n8n-io/n8n/issues/12643)) ([14f4bc7](https://github.com/n8n-io/n8n/commit/14f4bc769027789513808b4000444edf99dc5d1c)) +* **GraphQL Node:** Change default request format to json instead of graphql ([#11346](https://github.com/n8n-io/n8n/issues/11346)) ([c7c122f](https://github.com/n8n-io/n8n/commit/c7c122f9173df824cc1b5ab864333bffd0d31f82)) +* **Jira Software Node:** Get custom fields(RLC) in update operation for server deployment type ([#12719](https://github.com/n8n-io/n8n/issues/12719)) ([353df79](https://github.com/n8n-io/n8n/commit/353df7941117e20547cd4f3fc514979a54619720)) +* **n8n Form Node:** Remove the ability to change the formatting of dates ([#12666](https://github.com/n8n-io/n8n/issues/12666)) ([14904ff](https://github.com/n8n-io/n8n/commit/14904ff77951fef23eb789a43947492a4cd3fa20)) +* **OpenAI Chat Model Node:** Fix loading of custom models when using custom credential URL ([#12634](https://github.com/n8n-io/n8n/issues/12634)) ([7cc553e](https://github.com/n8n-io/n8n/commit/7cc553e3b277a16682bfca1ea08cb98178e38580)) +* **OpenAI Chat Model Node:** Restore default model value ([#12745](https://github.com/n8n-io/n8n/issues/12745)) ([d1b6692](https://github.com/n8n-io/n8n/commit/d1b6692736182fa2eab768ba3ad0adb8504ebbbd)) +* **Postgres Chat Memory Node:** Do not terminate the connection pool ([#12674](https://github.com/n8n-io/n8n/issues/12674)) ([e7f00bc](https://github.com/n8n-io/n8n/commit/e7f00bcb7f2dce66ca07a9322d50f96356c1a43d)) +* **Postgres Node:** Allow using composite key in upsert queries ([#12639](https://github.com/n8n-io/n8n/issues/12639)) ([83ce3a9](https://github.com/n8n-io/n8n/commit/83ce3a90963ba76601234f4314363a8ccc310f0f)) +* **Wait Node:** Fix for hasNextPage in waiting forms ([#12636](https://github.com/n8n-io/n8n/issues/12636)) ([652b8d1](https://github.com/n8n-io/n8n/commit/652b8d170b9624d47b5f2d8d679c165cc14ea548)) + + +### Features + +* Add credential only node for Microsoft Azure Monitor ([#12645](https://github.com/n8n-io/n8n/issues/12645)) ([6ef8882](https://github.com/n8n-io/n8n/commit/6ef8882a108c672ab097c9dd1c590d4e9e7f3bcc)) +* Add Miro credential only node ([#12746](https://github.com/n8n-io/n8n/issues/12746)) ([5b29086](https://github.com/n8n-io/n8n/commit/5b29086e2f9b7f638fac4440711f673438e57492)) +* Add SSM endpoint to AWS credentials ([#12212](https://github.com/n8n-io/n8n/issues/12212)) ([565c7b8](https://github.com/n8n-io/n8n/commit/565c7b8b9cfd3e10f6a2c60add96fea4c4d95d33)) +* **core:** Enable task runner by default ([#12726](https://github.com/n8n-io/n8n/issues/12726)) ([9e2a01a](https://github.com/n8n-io/n8n/commit/9e2a01aeaf36766a1cf7a1d9a4d6e02f45739bd3)) +* **editor:** Force final canvas v2 migration and remove switcher from UI ([#12717](https://github.com/n8n-io/n8n/issues/12717)) ([29335b9](https://github.com/n8n-io/n8n/commit/29335b9b6acf97c817bea70688e8a2786fbd8889)) +* **editor:** VariablesView Reskin - Add Filters for missing values ([#12611](https://github.com/n8n-io/n8n/issues/12611)) ([1eeb788](https://github.com/n8n-io/n8n/commit/1eeb788d327287d21eab7ad6f2156453ab7642c7)) +* **Jira Software Node:** Personal Access Token credential type ([#11038](https://github.com/n8n-io/n8n/issues/11038)) ([1c7a38f](https://github.com/n8n-io/n8n/commit/1c7a38f6bab108daa47401cd98c185590bf299a8)) +* **n8n Form Trigger Node:** Form Improvements ([#12590](https://github.com/n8n-io/n8n/issues/12590)) ([f167578](https://github.com/n8n-io/n8n/commit/f167578b3251e553a4d000e731e1bb60348916ad)) +* Synchronize deletions when pulling from source control ([#12170](https://github.com/n8n-io/n8n/issues/12170)) ([967ee4b](https://github.com/n8n-io/n8n/commit/967ee4b89b94b92fc3955c56bf4c9cca0bd64eac)) + + + +# [1.75.0](https://github.com/n8n-io/n8n/compare/n8n@1.74.0...n8n@1.75.0) (2025-01-15) + + +### Bug Fixes + +* **core:** AugmentObject should check for own propeties correctly ([#12534](https://github.com/n8n-io/n8n/issues/12534)) ([0cdf393](https://github.com/n8n-io/n8n/commit/0cdf39374305e6bbcedb047db7d3756168e6e89e)) +* **core:** Disallow code generation in task runner ([#12522](https://github.com/n8n-io/n8n/issues/12522)) ([35b6180](https://github.com/n8n-io/n8n/commit/35b618098b7d23e272bf77b55c172dbe531c821f)) +* **core:** Fix node exclusion on the frontend types ([#12544](https://github.com/n8n-io/n8n/issues/12544)) ([b2cbed9](https://github.com/n8n-io/n8n/commit/b2cbed9865888f6f3bc528984d4091d86a88f0d6)) +* **core:** Fix orchestration flow with expired license ([#12444](https://github.com/n8n-io/n8n/issues/12444)) ([ecff3b7](https://github.com/n8n-io/n8n/commit/ecff3b732a028d7225bfbed4ffc65dc20c4ed608)) +* **core:** Fix Sentry error reporting on task runners ([#12495](https://github.com/n8n-io/n8n/issues/12495)) ([88c0838](https://github.com/n8n-io/n8n/commit/88c0838dd72f11646bdb3586223d6c16631cccab)) +* **core:** Improve cyclic dependency check in the DI container ([#12600](https://github.com/n8n-io/n8n/issues/12600)) ([c3c4a20](https://github.com/n8n-io/n8n/commit/c3c4a200024fb08afb9380357d1490c6707c5ec3)) +* **core:** Only show personal credentials in the personal space ([#12433](https://github.com/n8n-io/n8n/issues/12433)) ([8a42d55](https://github.com/n8n-io/n8n/commit/8a42d55d91f4a37fff5669d52d52428b3a4ddd44)) +* **core:** Prefix package name in `supportedNodes` on generated types as well ([#12514](https://github.com/n8n-io/n8n/issues/12514)) ([4a1a999](https://github.com/n8n-io/n8n/commit/4a1a9993624c92dd81f5418f9268cb93878069ab)) +* **core:** Prevent prototype pollution in task runner ([#12588](https://github.com/n8n-io/n8n/issues/12588)) ([bdf266c](https://github.com/n8n-io/n8n/commit/bdf266cf55032d05641b20dce8804412dc93b6d5)) +* **core:** Prevent prototype pollution of internal classes in task runner ([#12610](https://github.com/n8n-io/n8n/issues/12610)) ([eceee7f](https://github.com/n8n-io/n8n/commit/eceee7f3f8899d200b1c5720087cc494eec22e6a)) +* **core:** Use timing safe function to compare runner auth tokens ([#12485](https://github.com/n8n-io/n8n/issues/12485)) ([8fab98f](https://github.com/n8n-io/n8n/commit/8fab98f3f1f767d05825d24cbf155d56375fdb3e)) +* **core:** Validate values which are intentionally 0 ([#12382](https://github.com/n8n-io/n8n/issues/12382)) ([562506e](https://github.com/n8n-io/n8n/commit/562506e92aeb26423145801bff80037e5ce2ac46)) +* Don't break oauth credentials when updating them and allow fixing broken oauth credentials by repeating the authorization flow ([#12563](https://github.com/n8n-io/n8n/issues/12563)) ([73897c7](https://github.com/n8n-io/n8n/commit/73897c7662a432834eb6f9d0f9ace8d986c1acb5)) +* **editor:** Don't show toolsUnused notice if run had errors ([#12529](https://github.com/n8n-io/n8n/issues/12529)) ([3ec5b28](https://github.com/n8n-io/n8n/commit/3ec5b2850c47057032e61c2acdbdfc1dcdd931f7)) +* **editor:** Ensure proper "AI Template" URL construction in node creator ([#12566](https://github.com/n8n-io/n8n/issues/12566)) ([13bf69f](https://github.com/n8n-io/n8n/commit/13bf69f75c67bc37a37013e776525768676a4b88)) +* **editor:** Fix NDV resize handle and scrollbar overlapping ([#12509](https://github.com/n8n-io/n8n/issues/12509)) ([c28f302](https://github.com/n8n-io/n8n/commit/c28f302c2f863bd7aa73ad52e5d040f927e33220)) +* **editor:** Fix parameter input validation ([#12532](https://github.com/n8n-io/n8n/issues/12532)) ([6711cbc](https://github.com/n8n-io/n8n/commit/6711cbcc641a2fc70f5c15a7e2dcc640a3f98b66)) +* **editor:** Fix selection rectangle context menu on new canvas ([#12584](https://github.com/n8n-io/n8n/issues/12584)) ([c8e3c53](https://github.com/n8n-io/n8n/commit/c8e3c5399efde93486c1dd5c373cb2c5ff8a0691)) +* **editor:** Fix the `openselectivenodecreator` custom action on new canvas ([#12580](https://github.com/n8n-io/n8n/issues/12580)) ([2110e9a](https://github.com/n8n-io/n8n/commit/2110e9a0513b8c36beb85302e0d38a2658ea5d6e)) +* **editor:** Fix workflow initilisation for test definition routes & add unit tests ([#12507](https://github.com/n8n-io/n8n/issues/12507)) ([2775f61](https://github.com/n8n-io/n8n/commit/2775f617ae5c267c0a1ce7a54d05d4077cdbc0f7)) +* **editor:** Make clicking item in RLC work the first time on small screens ([#12585](https://github.com/n8n-io/n8n/issues/12585)) ([479933f](https://github.com/n8n-io/n8n/commit/479933fbd5c88e783827960e018abb979de8a039)) +* **editor:** Make sure code editors work correctly in fullscreen ([#12597](https://github.com/n8n-io/n8n/issues/12597)) ([aa1f3a7](https://github.com/n8n-io/n8n/commit/aa1f3a7d989883d55df3777775b8d7d336f6e3b7)) +* **editor:** Override selected nodes on single click without Meta/Ctrl key ([#12549](https://github.com/n8n-io/n8n/issues/12549)) ([02c2d5e](https://github.com/n8n-io/n8n/commit/02c2d5e71d15b9292fddd585f47bd8334da468c5)) +* **editor:** Show NDV errors when opening existing nodes with errors ([#12567](https://github.com/n8n-io/n8n/issues/12567)) ([bee7267](https://github.com/n8n-io/n8n/commit/bee7267fe38ab12a79fa4ec0e775f45d98d48aa5)) +* **editor:** Swap Activate/Deactivate texts in FloatingToolbar ([#12526](https://github.com/n8n-io/n8n/issues/12526)) ([44679b4](https://github.com/n8n-io/n8n/commit/44679b42aa1e14bc7069bee47d0a91ca84b1dba4)) +* **editor:** Update filter and feedback for source control ([#12504](https://github.com/n8n-io/n8n/issues/12504)) ([865fc21](https://github.com/n8n-io/n8n/commit/865fc21276727e8d88ccee0355147904b81c4421)) +* **editor:** Update selected node when navigating via flowing nodes ([#12581](https://github.com/n8n-io/n8n/issues/12581)) ([88659d8](https://github.com/n8n-io/n8n/commit/88659d8a2901786c894902e19466f395bcdaab8e)) +* **Google Calendar Node:** Updates and fixes ([#10715](https://github.com/n8n-io/n8n/issues/10715)) ([7227a29](https://github.com/n8n-io/n8n/commit/7227a29845fd178ced4d281597c62e7a03245456)) +* **Spotify Node:** Fix issue with null values breaking the response ([#12080](https://github.com/n8n-io/n8n/issues/12080)) ([a56a462](https://github.com/n8n-io/n8n/commit/a56a46259d257003c813103578260d625b3f17dd)) + + +### Features + +* **editor:** Make node credential select searchable ([#12497](https://github.com/n8n-io/n8n/issues/12497)) ([91277c4](https://github.com/n8n-io/n8n/commit/91277c44f1cf3f334b3b50d47d7dcc79b11c7c63)) +* **editor:** Persist sidebar collapsed status preference ([#12505](https://github.com/n8n-io/n8n/issues/12505)) ([dba7d46](https://github.com/n8n-io/n8n/commit/dba7d46f3ec91d26a597a50dede7b6ca292c728f)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ed101af7d..5e495fda9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,7 +85,7 @@ This automatically sets up file-links between modules which depend on each other We recommend enabling [Node.js corepack](https://nodejs.org/docs/latest-v16.x/api/corepack.html) with `corepack enable`. -With Node.js v16.17 or newer, you can install the latest version of pnpm: `corepack prepare pnpm@latest --activate`. If you use an older version install at least version 7.18 of pnpm via: `corepack prepare pnpm@7.18.0 --activate`. +With Node.js v16.17 or newer, you can install the latest version of pnpm: `corepack prepare pnpm@latest --activate`. If you use an older version install at least version 9.15 of pnpm via: `corepack prepare pnpm@9.15.5 --activate`. **IMPORTANT**: If you have installed Node.js via homebrew, you'll need to run `brew install corepack`, since homebrew explicitly removes `npm` and `corepack` from [the `node` formula](https://github.com/Homebrew/homebrew-core/blob/master/Formula/node.rb#L66). diff --git a/codecov.yml b/codecov.yml index 362ebcb760..dfcd2b0d9c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -46,6 +46,7 @@ component_management: - packages/@n8n/codemirror-lang/** - packages/design-system/** - packages/editor-ui/** + - packages/frontend/** - component_id: nodes_packages name: Nodes paths: diff --git a/cypress/composables/modals/credential-modal.ts b/cypress/composables/modals/credential-modal.ts index 53ba0c3a28..4e6533300a 100644 --- a/cypress/composables/modals/credential-modal.ts +++ b/cypress/composables/modals/credential-modal.ts @@ -2,6 +2,8 @@ * Getters */ +import { clearNotifications } from '../../pages/notifications'; + export function getCredentialConnectionParameterInputs() { return cy.getByTestId('credential-connection-parameter'); } @@ -55,5 +57,6 @@ export function setCredentialValues(values: Record, save = true) if (save) { saveCredential(); closeCredentialModal(); + clearNotifications(); } } diff --git a/cypress/composables/ndv.ts b/cypress/composables/ndv.ts index 90146ab374..c70e0b20e8 100644 --- a/cypress/composables/ndv.ts +++ b/cypress/composables/ndv.ts @@ -2,7 +2,11 @@ * Getters */ -import { getVisibleSelect } from '../utils/popper'; +import { getVisiblePopper, getVisibleSelect } from '../utils/popper'; + +export function getNdvContainer() { + return cy.getByTestId('ndv'); +} export function getCredentialSelect(eq = 0) { return cy.getByTestId('node-credentials-select').eq(eq); @@ -36,6 +40,18 @@ export function getOutputPanel() { return cy.getByTestId('output-panel'); } +export function getFixedCollection(collectionName: string) { + return cy.getByTestId(`fixed-collection-${collectionName}`); +} + +export function getResourceLocator(paramName: string) { + return cy.getByTestId(`resource-locator-${paramName}`); +} + +export function getResourceLocatorInput(paramName: string) { + return getResourceLocator(paramName).find('[data-test-id="rlc-input-container"]'); +} + export function getOutputPanelDataContainer() { return getOutputPanel().getByTestId('ndv-data-container'); } @@ -84,6 +100,30 @@ export function getOutputPanelRelatedExecutionLink() { return getOutputPanel().getByTestId('related-execution-link'); } +export function getNodeOutputHint() { + return cy.getByTestId('ndv-output-run-node-hint'); +} + +export function getWorkflowCards() { + return cy.getByTestId('resources-list-item'); +} + +export function getWorkflowCard(workflowName: string) { + return getWorkflowCards().contains(workflowName).parents('[data-test-id="resources-list-item"]'); +} + +export function getWorkflowCardContent(workflowName: string) { + return getWorkflowCard(workflowName).findChildByTestId('card-content'); +} + +export function getNodeRunInfoStale() { + return cy.getByTestId('node-run-info-stale'); +} + +export function getNodeOutputErrorMessage() { + return getOutputPanel().findChildByTestId('node-error-message'); +} + /** * Actions */ @@ -110,12 +150,20 @@ export function clickExecuteNode() { getExecuteNodeButton().click(); } +export function clickResourceLocatorInput(paramName: string) { + getResourceLocatorInput(paramName).click(); +} + export function setParameterInputByName(name: string, value: string) { getParameterInputByName(name).clear().type(value); } -export function toggleParameterCheckboxInputByName(name: string) { - getParameterInputByName(name).find('input[type="checkbox"]').realClick(); +export function checkParameterCheckboxInputByName(name: string) { + getParameterInputByName(name).find('input[type="checkbox"]').check({ force: true }); +} + +export function uncheckParameterCheckboxInputByName(name: string) { + getParameterInputByName(name).find('input[type="checkbox"]').uncheck({ force: true }); } export function setParameterSelectByContent(name: string, content: string) { @@ -127,3 +175,86 @@ export function changeOutputRunSelector(runName: string) { getOutputRunSelector().click(); getVisibleSelect().find('.el-select-dropdown__item').contains(runName).click(); } + +export function addItemToFixedCollection(collectionName: string) { + getFixedCollection(collectionName).getByTestId('fixed-collection-add').click(); +} + +export function typeIntoFixedCollectionItem(collectionName: string, index: number, value: string) { + getFixedCollection(collectionName).within(() => + cy.getByTestId('parameter-input').eq(index).type(value), + ); +} + +export function selectResourceLocatorItem( + resourceLocator: string, + index: number, + expectedText: string, +) { + clickResourceLocatorInput(resourceLocator); + + getVisiblePopper().findChildByTestId('rlc-item').eq(0).should('exist'); + getVisiblePopper() + .findChildByTestId('rlc-item') + .eq(index) + .find('span') + .should('contain.text', expectedText) + .click(); +} + +export function clickWorkflowCardContent(workflowName: string) { + getWorkflowCardContent(workflowName).click(); +} + +export function assertNodeOutputHintExists() { + getNodeOutputHint().should('exist'); +} + +export function assertNodeOutputErrorMessageExists() { + return getNodeOutputErrorMessage().should('exist'); +} + +// Note that this only validates the expectedContent is *included* in the output table +export function assertOutputTableContent(expectedContent: unknown[][]) { + for (const [i, row] of expectedContent.entries()) { + for (const [j, value] of row.entries()) { + // + 1 to skip header + getOutputTbodyCell(1 + i, j).should('have.text', value); + } + } +} + +export function populateMapperFields(fields: ReadonlyArray<[string, string]>) { + for (const [name, value] of fields) { + getParameterInputByName(name).type(value); + + // Click on a parent to dismiss the pop up which hides the field below. + getParameterInputByName(name).parent().parent().parent().parent().click('topLeft'); + } +} + +/** + * Populate multiValue fixedCollections. Only supports fixedCollections for which all fields can be defined via keyboard typing + * + * @param items - 2D array of items to populate, i.e. [["myField1", "String"], ["myField2", "Number"]] + * @param collectionName - name of the fixedCollection to populate + * @param offset - amount of 'parameter-input's before start, e.g. from a controlling dropdown that makes the fields appear + * @returns + */ +export function populateFixedCollection( + items: readonly T[], + collectionName: string, + offset: number = 0, +) { + if (items.length === 0) return; + const n = items[0].length; + for (const [i, params] of items.entries()) { + addItemToFixedCollection(collectionName); + for (const [j, param] of params.entries()) { + getFixedCollection(collectionName) + .getByTestId('parameter-input') + .eq(offset + i * n + j) + .type(`${param}{downArrow}{enter}`); + } + } +} diff --git a/cypress/composables/webhooks.ts b/cypress/composables/webhooks.ts new file mode 100644 index 0000000000..8ad5dc6861 --- /dev/null +++ b/cypress/composables/webhooks.ts @@ -0,0 +1,81 @@ +import { BACKEND_BASE_URL } from '../constants'; +import { NDV, WorkflowPage } from '../pages'; +import { getVisibleSelect } from '../utils'; + +export const waitForWebhook = 500; + +export interface SimpleWebhookCallOptions { + method: string; + webhookPath: string; + responseCode?: number; + respondWith?: string; + executeNow?: boolean; + responseData?: string; + authentication?: string; +} + +const workflowPage = new WorkflowPage(); +const ndv = new NDV(); + +export const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { + const { + authentication, + method, + webhookPath, + responseCode, + respondWith, + responseData, + executeNow = true, + } = options; + + workflowPage.actions.addInitialNodeToCanvas('Webhook'); + workflowPage.actions.openNode('Webhook'); + + cy.getByTestId('parameter-input-httpMethod').click(); + getVisibleSelect().find('.option-headline').contains(method).click(); + cy.getByTestId('parameter-input-path') + .find('.parameter-input') + .find('input') + .clear() + .type(webhookPath); + + if (authentication) { + cy.getByTestId('parameter-input-authentication').click(); + getVisibleSelect().find('.option-headline').contains(authentication).click(); + } + + if (responseCode) { + cy.get('.param-options').click(); + getVisibleSelect().contains('Response Code').click(); + cy.get('.parameter-item-wrapper > .parameter-input-list-wrapper').children().click(); + getVisibleSelect().contains('201').click(); + } + + if (respondWith) { + cy.getByTestId('parameter-input-responseMode').click(); + getVisibleSelect().find('.option-headline').contains(respondWith).click(); + } + + if (responseData) { + cy.getByTestId('parameter-input-responseData').click(); + getVisibleSelect().find('.option-headline').contains(responseData).click(); + } + + const callEndpoint = (fn: (response: Cypress.Response) => void) => { + cy.request(method, `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then(fn); + }; + + if (executeNow) { + ndv.actions.execute(); + cy.wait(waitForWebhook); + + callEndpoint((response) => { + expect(response.status).to.eq(200); + ndv.getters.outputPanel().contains('headers'); + }); + } + + return { + callEndpoint, + }; +}; diff --git a/cypress/composables/workflow.ts b/cypress/composables/workflow.ts index e8e402dd5f..7d783c1d3c 100644 --- a/cypress/composables/workflow.ts +++ b/cypress/composables/workflow.ts @@ -1,12 +1,14 @@ import { getManualChatModal } from './modals/chat-modal'; import { clickGetBackToCanvas, getParameterInputByName } from './ndv'; import { ROUTES } from '../constants'; +import type { OpenContextMenuOptions } from '../types'; /** * Types */ export type EndpointType = + | 'main' | 'ai_chain' | 'ai_document' | 'ai_embedding' @@ -23,9 +25,75 @@ export type EndpointType = * Getters */ -export function getAddInputEndpointByType(nodeName: string, endpointType: EndpointType) { - return cy.get( - `.add-input-endpoint[data-jtk-scope-${endpointType}][data-endpoint-name="${nodeName}"]`, +export function getCanvas() { + return cy.getByTestId('canvas'); +} + +export function getCanvasPane() { + return cy.ifCanvasVersion( + () => cy.getByTestId('node-view-background'), + () => getCanvas().find('.vue-flow__pane'), + ); +} + +export function getContextMenu() { + return cy.getByTestId('context-menu').find('.el-dropdown-menu'); +} + +export function getContextMenuAction(action: string) { + return cy.getByTestId(`context-menu-item-${action}`); +} + +export function getInputPlusHandle(nodeName: string) { + return cy.ifCanvasVersion( + () => cy.get(`.add-input-endpoint[data-endpoint-name="${nodeName}"]`), + () => + cy.get( + `[data-test-id="canvas-node-input-handle"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"]`, + ), + ); +} + +export function getInputPlusHandleByType(nodeName: string, endpointType: EndpointType) { + return cy.ifCanvasVersion( + () => + cy.get( + `.add-input-endpoint[data-jtk-scope-${endpointType}][data-endpoint-name="${nodeName}"]`, + ), + () => + cy.get( + `[data-test-id="canvas-node-input-handle"][data-connection-type="${endpointType}"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"]`, + ), + ); +} + +export function getOutputHandle(nodeName: string) { + return cy.ifCanvasVersion( + () => cy.get(`.add-output-endpoint[data-endpoint-name="${nodeName}"]`), + () => cy.get(`[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"]`), + ); +} + +export function getOutputPlusHandle(nodeName: string) { + return cy.ifCanvasVersion( + () => cy.get(`.add-output-endpoint[data-endpoint-name="${nodeName}"]`), + () => + cy.get( + `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"]`, + ), + ); +} + +export function getOutputPlusHandleByType(nodeName: string, endpointType: EndpointType) { + return cy.ifCanvasVersion( + () => + cy.get( + `.add-output-endpoint[data-jtk-scope-${endpointType}][data-endpoint-name="${nodeName}"]`, + ), + () => + cy.get( + `[data-test-id="canvas-node-output-handle"][data-connection-type="${endpointType}"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"]`, + ), ); } @@ -33,8 +101,8 @@ export function getNodeCreatorItems() { return cy.getByTestId('item-iterator-item'); } -export function getExecuteWorkflowButton() { - return cy.getByTestId('execute-workflow-button'); +export function getExecuteWorkflowButton(triggerNodeName?: string) { + return cy.getByTestId(`execute-workflow-button${triggerNodeName ? `-${triggerNodeName}` : ''}`); } export function getManualChatButton() { @@ -52,6 +120,13 @@ export function getNodeByName(name: string) { ); } +export function getNodeRenderedTypeByName(name: string) { + return cy.ifCanvasVersion( + () => getNodeByName(name), + () => getNodeByName(name).find('[data-canvas-node-render-type]'), + ); +} + export function getWorkflowHistoryCloseButton() { return cy.getByTestId('workflow-history-close-button'); } @@ -63,10 +138,24 @@ export function disableNode(name: string) { } export function getConnectionBySourceAndTarget(source: string, target: string) { + return cy.ifCanvasVersion( + () => + cy + .get('.jtk-connector') + .filter(`[data-source-node="${source}"][data-target-node="${target}"]`) + .eq(0), + () => + cy + .getByTestId('edge') + .filter(`[data-source-node-name="${source}"][data-target-node-name="${target}"]`) + .eq(0), + ); +} + +export function getConnectionLabelBySourceAndTarget(source: string, target: string) { return cy - .get('.jtk-connector') - .filter(`[data-source-node="${source}"][data-target-node="${target}"]`) - .eq(0); + .getByTestId('edge-label') + .filter(`[data-source-node-name="${source}"][data-target-node-name="${target}"]`); } export function getNodeCreatorSearchBar() { @@ -78,10 +167,7 @@ export function getNodeCreatorPlusButton() { } export function getCanvasNodes() { - return cy.ifCanvasVersion( - () => cy.getByTestId('canvas-node'), - () => cy.getByTestId('canvas-node').not('[data-node-type="n8n-nodes-internal.addNodes"]'), - ); + return cy.getByTestId('canvas-node'); } export function getCanvasNodeByName(nodeName: string) { @@ -141,7 +227,7 @@ function connectNodeToParent( parentNodeName: string, exactMatch = false, ) { - getAddInputEndpointByType(parentNodeName, endpointType).click({ force: true }); + getInputPlusHandleByType(parentNodeName, endpointType).click({ force: true }); if (exactMatch) { getNodeCreatorItems() .contains(new RegExp('^' + nodeName + '$', 'g')) @@ -158,7 +244,19 @@ export function addSupplementalNodeToParent( exactMatch = false, ) { connectNodeToParent(nodeName, endpointType, parentNodeName, exactMatch); - getConnectionBySourceAndTarget(parentNodeName, nodeName).should('exist'); + + cy.ifCanvasVersion( + () => { + getConnectionBySourceAndTarget(parentNodeName, nodeName).should('exist'); + }, + () => { + if (endpointType === 'main') { + getConnectionBySourceAndTarget(parentNodeName, nodeName).should('exist'); + } else { + getConnectionBySourceAndTarget(nodeName, parentNodeName).should('exist'); + } + }, + ); } export function addLanguageModelNodeToParent( @@ -196,8 +294,8 @@ export function addRetrieverNodeToParent(nodeName: string, parentNodeName: strin addSupplementalNodeToParent(nodeName, 'ai_retriever', parentNodeName); } -export function clickExecuteWorkflowButton() { - getExecuteWorkflowButton().click(); +export function clickExecuteWorkflowButton(triggerNodeName?: string) { + getExecuteWorkflowButton(triggerNodeName).click(); } export function clickManualChatButton() { @@ -229,3 +327,34 @@ export function deleteNode(name: string) { getCanvasNodeByName(name).first().click(); cy.get('body').type('{del}'); } + +export function openContextMenu( + nodeName?: string, + { method = 'right-click', anchor = 'center' }: OpenContextMenuOptions = {}, +) { + let target; + if (nodeName) { + target = + method === 'right-click' ? getNodeRenderedTypeByName(nodeName) : getNodeByName(nodeName); + } else { + target = getCanvasPane(); + } + + if (method === 'right-click') { + target.rightclick(nodeName ? anchor : 'topLeft', { force: true }); + } else { + target.realHover(); + target.find('[data-test-id="overflow-node-button"]').click({ force: true }); + } + + cy.ifCanvasVersion( + () => {}, + () => { + getContextMenu().should('be.visible'); + }, + ); +} + +export function clickContextMenuAction(action: string) { + getContextMenuAction(action).click(); +} diff --git a/cypress/composables/workflowsPage.ts b/cypress/composables/workflowsPage.ts new file mode 100644 index 0000000000..c7bcf39888 --- /dev/null +++ b/cypress/composables/workflowsPage.ts @@ -0,0 +1,15 @@ +/** + * Getters + */ + +export function getWorkflowsPageUrl() { + return '/home/workflows'; +} + +/** + * Actions + */ + +export function visitWorkflowsPage() { + cy.visit(getWorkflowsPageUrl()); +} diff --git a/cypress/e2e/10-undo-redo.cy.ts b/cypress/e2e/10-undo-redo.cy.ts index f54c2de9fa..2931897f03 100644 --- a/cypress/e2e/10-undo-redo.cy.ts +++ b/cypress/e2e/10-undo-redo.cy.ts @@ -1,16 +1,15 @@ +import { getCanvasNodes } from '../composables/workflow'; import { SCHEDULE_TRIGGER_NODE_NAME, CODE_NODE_NAME, SET_NODE_NAME, - EDIT_FIELDS_SET_NODE_NAME, + MANUAL_TRIGGER_NODE_NAME, + MANUAL_TRIGGER_NODE_DISPLAY_NAME, } from '../constants'; import { MessageBox as MessageBoxClass } from '../pages/modals/message-box'; import { NDV } from '../pages/ndv'; import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; -// Suite-specific constants -const CODE_NODE_NEW_NAME = 'Something else'; - const WorkflowPage = new WorkflowPageClass(); const messageBox = new MessageBoxClass(); const ndv = new NDV(); @@ -20,40 +19,6 @@ describe('Undo/Redo', () => { WorkflowPage.actions.visit(); }); - // FIXME: Canvas V2: Fix redo connections - it('should undo/redo adding node in the middle', () => { - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.actions.addNodeBetweenNodes( - SCHEDULE_TRIGGER_NODE_NAME, - CODE_NODE_NAME, - SET_NODE_NAME, - ); - WorkflowPage.actions.zoomToFit(); - WorkflowPage.getters.canvasNodeByName('Code').then(($codeNode) => { - const cssLeft = parseInt($codeNode.css('left')); - const cssTop = parseInt($codeNode.css('top')); - - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters.canvasNodes().should('have.have.length', 2); - WorkflowPage.getters.nodeConnections().should('have.length', 1); - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters.canvasNodes().should('have.have.length', 1); - WorkflowPage.getters.nodeConnections().should('have.length', 0); - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.canvasNodes().should('have.have.length', 2); - WorkflowPage.getters.nodeConnections().should('have.length', 1); - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.canvasNodes().should('have.have.length', 3); - WorkflowPage.getters.nodeConnections().should('have.length', 2); - // Last node should be added back to original position - WorkflowPage.getters - .canvasNodeByName('Code') - .should('have.css', 'left', cssLeft + 'px') - .should('have.css', 'top', cssTop + 'px'); - }); - }); - it('should undo/redo deleting node using context menu', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -115,34 +80,60 @@ describe('Undo/Redo', () => { WorkflowPage.getters.nodeConnections().should('have.length', 0); }); - // FIXME: Canvas V2: Fix moving of nodes via e2e tests it('should undo/redo moving nodes', () => { - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); + WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); + WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.getters.canvasNodeByName(CODE_NODE_NAME).then(($node) => { - const initialPosition = $node.position(); - cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { clickToFinish: true }); - WorkflowPage.getters.canvasNodeByName(CODE_NODE_NAME).then(($node) => { - const cssLeft = parseInt($node.css('left')); - const cssTop = parseInt($node.css('top')); - expect(cssLeft).to.be.greaterThan(initialPosition.left); - expect(cssTop).to.be.greaterThan(initialPosition.top); - }); + WorkflowPage.actions.zoomToFit(); - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters - .canvasNodeByName(CODE_NODE_NAME) - .should('have.css', 'left', `${initialPosition.left}px`) - .should('have.css', 'top', `${initialPosition.top}px`); - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.canvasNodeByName(CODE_NODE_NAME).then(($node) => { - const cssLeft = parseInt($node.css('left')); - const cssTop = parseInt($node.css('top')); - expect(cssLeft).to.be.greaterThan(initialPosition.left); - expect(cssTop).to.be.greaterThan(initialPosition.top); + getCanvasNodes() + .last() + .then(($node) => { + const { x: x1, y: y1 } = $node[0].getBoundingClientRect(); + + cy.ifCanvasVersion( + () => { + cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { + clickToFinish: true, + }); + }, + () => { + cy.drag(getCanvasNodes().last(), [50, 150], { + realMouse: true, + abs: true, + }); + }, + ); + + getCanvasNodes() + .last() + .then(($node) => { + const { x: x2, y: y2 } = $node[0].getBoundingClientRect(); + expect(x2).to.be.greaterThan(x1); + expect(y2).to.be.greaterThan(y1); + }); + + WorkflowPage.actions.hitUndo(); + + getCanvasNodes() + .last() + .then(($node) => { + const { x: x3, y: y3 } = $node[0].getBoundingClientRect(); + expect(x3).to.equal(x1); + expect(y3).to.equal(y1); + }); + + WorkflowPage.actions.hitRedo(); + + getCanvasNodes() + .last() + .then(($node) => { + const { x: x4, y: y4 } = $node[0].getBoundingClientRect(); + expect(x4).to.be.greaterThan(x1); + expect(y4).to.be.greaterThan(y1); + }); }); - }); }); it('should undo/redo deleting a connection using context menu', () => { @@ -155,17 +146,6 @@ describe('Undo/Redo', () => { WorkflowPage.actions.hitRedo(); WorkflowPage.getters.nodeConnections().should('have.length', 0); }); - // FIXME: Canvas V2: Fix disconnecting by moving - it('should undo/redo deleting a connection by moving it away', () => { - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - cy.drag('.rect-input-endpoint.jtk-endpoint-connected', [0, -100]); - WorkflowPage.getters.nodeConnections().should('have.length', 0); - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters.nodeConnections().should('have.length', 1); - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.nodeConnections().should('have.length', 0); - }); it('should undo/redo disabling a node using context menu', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); @@ -204,23 +184,6 @@ describe('Undo/Redo', () => { WorkflowPage.getters.disabledNodes().should('have.length', 2); }); - // FIXME: Canvas V2: Fix undo renaming node - it('should undo/redo renaming node using keyboard shortcut', () => { - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.getters.canvasNodes().last().click(); - cy.get('body').trigger('keydown', { key: 'F2' }); - cy.get('.rename-prompt').should('be.visible'); - cy.get('body').type(CODE_NODE_NEW_NAME); - cy.get('body').type('{enter}'); - WorkflowPage.actions.hitUndo(); - cy.get('body').type('{esc}'); - WorkflowPage.getters.canvasNodeByName(CODE_NODE_NAME).should('exist'); - WorkflowPage.actions.hitRedo(); - cy.get('body').type('{esc}'); - WorkflowPage.getters.canvasNodeByName(CODE_NODE_NEW_NAME).should('exist'); - }); - it('should undo/redo duplicating a node', () => { WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -243,77 +206,6 @@ describe('Undo/Redo', () => { }); }); - // FIXME: Canvas V2: Figure out why moving doesn't work from e2e - it('should undo/redo multiple steps', () => { - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - // WorkflowPage.actions.addNodeToCanvas(SET_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - WorkflowPage.actions.zoomToFit(); - - // Disable last node - WorkflowPage.getters.canvasNodes().last().click(); - WorkflowPage.actions.hitDisableNodeShortcut(); - - // Move first one - WorkflowPage.actions - .getNodePosition(WorkflowPage.getters.canvasNodes().first()) - .then((initialPosition) => { - WorkflowPage.getters.canvasNodes().first().click(); - cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { - clickToFinish: true, - }); - WorkflowPage.getters - .canvasNodes() - .first() - .then(($node) => { - const cssLeft = parseInt($node.css('left')); - const cssTop = parseInt($node.css('top')); - expect(cssLeft).to.be.greaterThan(initialPosition.left); - expect(cssTop).to.be.greaterThan(initialPosition.top); - }); - - // Delete the set node - WorkflowPage.getters.canvasNodeByName(EDIT_FIELDS_SET_NODE_NAME).click().click(); - cy.get('body').type('{backspace}'); - - // First undo: Should return deleted node - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters.canvasNodes().should('have.length', 4); - WorkflowPage.getters.nodeConnections().should('have.length', 3); - // Second undo: Should move first node to it's original position - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters - .canvasNodes() - .first() - .should('have.css', 'left', `${initialPosition.left}px`) - .should('have.css', 'top', `${initialPosition.top}px`); - // Third undo: Should enable last node - WorkflowPage.actions.hitUndo(); - WorkflowPage.getters.disabledNodes().should('have.length', 0); - - // First redo: Should disable last node - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.disabledNodes().should('have.length', 1); - // Second redo: Should move the first node - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters - .canvasNodes() - .first() - .then(($node) => { - const cssLeft = parseInt($node.css('left')); - const cssTop = parseInt($node.css('top')); - expect(cssLeft).to.be.greaterThan(initialPosition.left); - expect(cssTop).to.be.greaterThan(initialPosition.top); - }); - // Third redo: Should delete the Set node - WorkflowPage.actions.hitRedo(); - WorkflowPage.getters.canvasNodes().should('have.length', 3); - WorkflowPage.getters.nodeConnections().should('have.length', 2); - }); - }); - it('should be able to copy and paste pinned data nodes in workflows with dynamic Switch node', () => { cy.fixture('Test_workflow_form_switch.json').then((data) => { cy.get('body').paste(JSON.stringify(data)); diff --git a/cypress/e2e/11-inline-expression-editor.cy.ts b/cypress/e2e/11-inline-expression-editor.cy.ts index a762135a65..e35842293e 100644 --- a/cypress/e2e/11-inline-expression-editor.cy.ts +++ b/cypress/e2e/11-inline-expression-editor.cy.ts @@ -129,7 +129,7 @@ describe('Inline expression editor', () => { // Run workflow ndv.actions.close(); - WorkflowPage.actions.executeNode('No Operation', { anchor: 'topLeft' }); + WorkflowPage.actions.executeNode('No Operation, do nothing', { anchor: 'topLeft' }); WorkflowPage.actions.openNode('Hacker News'); WorkflowPage.actions.openInlineExpressionEditor(); diff --git a/cypress/e2e/12-canvas-actions.cy.ts b/cypress/e2e/12-canvas-actions.cy.ts index e9244a1d12..f63c85dc49 100644 --- a/cypress/e2e/12-canvas-actions.cy.ts +++ b/cypress/e2e/12-canvas-actions.cy.ts @@ -4,9 +4,9 @@ import { CODE_NODE_NAME, SCHEDULE_TRIGGER_NODE_NAME, EDIT_FIELDS_SET_NODE_NAME, - IF_NODE_NAME, HTTP_REQUEST_NODE_NAME, } from './../constants'; +import { getCanvasPane } from '../composables/workflow'; import { successToast } from '../pages/notifications'; import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; @@ -16,64 +16,12 @@ describe('Canvas Actions', () => { WorkflowPage.actions.visit(); }); - // FIXME: Canvas V2: Missing execute button if no nodes - it('should render canvas', () => { - WorkflowPage.getters.nodeViewRoot().should('be.visible'); - WorkflowPage.getters.canvasPlusButton().should('be.visible'); - WorkflowPage.getters.zoomToFitButton().should('be.visible'); - WorkflowPage.getters.zoomInButton().should('be.visible'); - WorkflowPage.getters.zoomOutButton().should('be.visible'); - WorkflowPage.getters.executeWorkflowButton().should('be.visible'); - }); - - // FIXME: Canvas V2: Fix changing of connection - it('should connect and disconnect a simple node', () => { - WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); - WorkflowPage.getters.nodeViewBackground().click(600, 200, { force: true }); - WorkflowPage.getters.nodeConnections().should('have.length', 1); - - WorkflowPage.getters.nodeViewBackground().click(600, 400, { force: true }); - WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); - - // Change connection from Set to Set1 - cy.draganddrop( - WorkflowPage.getters.getEndpointSelector('input', EDIT_FIELDS_SET_NODE_NAME), - WorkflowPage.getters.getEndpointSelector('input', `${EDIT_FIELDS_SET_NODE_NAME}1`), - ); - - WorkflowPage.getters - .getConnectionBetweenNodes(MANUAL_TRIGGER_NODE_DISPLAY_NAME, `${EDIT_FIELDS_SET_NODE_NAME}1`) - .should('be.visible'); - - WorkflowPage.getters.nodeConnections().should('have.length', 1); - // Disconnect Set1 - cy.drag( - WorkflowPage.getters.getEndpointSelector('input', `${EDIT_FIELDS_SET_NODE_NAME}1`), - [-200, 100], - ); - WorkflowPage.getters.nodeConnections().should('have.length', 0); - }); - it('should add first step', () => { WorkflowPage.getters.canvasPlusButton().should('be.visible'); WorkflowPage.actions.addInitialNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodes().should('have.length', 1); }); - it('should add a node via plus endpoint drag', () => { - WorkflowPage.getters.canvasPlusButton().should('be.visible'); - WorkflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME, true); - - cy.drag( - WorkflowPage.getters.getEndpointSelector('plus', SCHEDULE_TRIGGER_NODE_NAME), - [100, 100], - ); - - WorkflowPage.getters.nodeCreatorSearchBar().should('be.visible'); - WorkflowPage.actions.addNodeToCanvas(IF_NODE_NAME, false); - WorkflowPage.getters.nodeViewBackground().click({ force: true }); - }); - it('should add a connected node using plus endpoint', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodePlusEndpointByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); @@ -116,7 +64,7 @@ describe('Canvas Actions', () => { it('should add disconnected node if nothing is selected', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); // Deselect nodes - WorkflowPage.getters.nodeView().click({ force: true }); + getCanvasPane().click({ force: true }); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); WorkflowPage.getters.canvasNodes().should('have.length', 2); WorkflowPage.getters.nodeConnections().should('have.length', 0); @@ -166,15 +114,6 @@ describe('Canvas Actions', () => { WorkflowPage.getters.nodeConnections().should('have.length', 0); }); - // FIXME: Canvas V2: Fix disconnecting of connection by dragging it - it('should delete a connection by moving it away from endpoint', () => { - WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); - WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); - cy.drag(WorkflowPage.getters.getEndpointSelector('input', CODE_NODE_NAME), [0, -100]); - WorkflowPage.getters.nodeConnections().should('have.length', 0); - }); - describe('Node hover actions', () => { it('should execute node', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); @@ -184,7 +123,11 @@ describe('Canvas Actions', () => { .last() .findChildByTestId('execute-node-button') .click({ force: true }); + + successToast().should('have.length', 1); + WorkflowPage.actions.executeNode(CODE_NODE_NAME); + successToast().should('have.length', 2); successToast().should('contain.text', 'Node executed successfully'); }); @@ -235,7 +178,6 @@ describe('Canvas Actions', () => { WorkflowPage.getters.selectedNodes().should('have.length', 0); }); - // FIXME: Canvas V2: Selection via arrow keys is broken it('should select nodes using arrow keys', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); @@ -259,7 +201,6 @@ describe('Canvas Actions', () => { ); }); - // FIXME: Canvas V2: Selection via shift and arrow keys is broken it('should select nodes using shift and arrow keys', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); @@ -268,31 +209,4 @@ describe('Canvas Actions', () => { cy.get('body').type('{shift}', { release: false }).type('{leftArrow}'); WorkflowPage.getters.selectedNodes().should('have.length', 2); }); - - // FIXME: Canvas V2: Fix select & deselect - it('should not break lasso selection when dragging node action buttons', () => { - WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - WorkflowPage.getters - .canvasNodes() - .last() - .findChildByTestId('execute-node-button') - .as('executeNodeButton'); - cy.drag('@executeNodeButton', [200, 200]); - WorkflowPage.actions.testLassoSelection([100, 100], [200, 200]); - }); - - // FIXME: Canvas V2: Fix select & deselect - it('should not break lasso selection with multiple clicks on node action buttons', () => { - WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); - WorkflowPage.actions.testLassoSelection([100, 100], [200, 200]); - WorkflowPage.getters.canvasNodes().last().as('lastNode'); - cy.get('@lastNode').findChildByTestId('execute-node-button').as('executeNodeButton'); - for (let i = 0; i < 20; i++) { - cy.get('@lastNode').realHover(); - cy.get('@executeNodeButton').should('be.visible'); - cy.get('@executeNodeButton').realTouch(); - cy.getByTestId('execute-workflow-button').realHover(); - WorkflowPage.actions.testLassoSelection([100, 100], [200, 200]); - } - }); }); diff --git a/cypress/e2e/12-canvas.cy.ts b/cypress/e2e/12-canvas.cy.ts index ecfb325de2..be423344fb 100644 --- a/cypress/e2e/12-canvas.cy.ts +++ b/cypress/e2e/12-canvas.cy.ts @@ -7,9 +7,17 @@ import { SWITCH_NODE_NAME, MERGE_NODE_NAME, } from './../constants'; +import { + clickContextMenuAction, + getCanvasNodeByName, + getCanvasNodes, + getConnectionBySourceAndTarget, + getConnectionLabelBySourceAndTarget, + getOutputPlusHandle, + openContextMenu, +} from '../composables/workflow'; import { NDV, WorkflowExecutionsTab } from '../pages'; import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; -import { isCanvasV2 } from '../utils/workflowUtils'; const WorkflowPage = new WorkflowPageClass(); const ExecutionsTab = new WorkflowExecutionsTab(); @@ -20,8 +28,6 @@ const ZOOM_IN_X2_FACTOR = 1.5625; // Zoom in factor after two clicks const ZOOM_OUT_X1_FACTOR = 0.8; const ZOOM_OUT_X2_FACTOR = 0.64; -const PINCH_ZOOM_IN_FACTOR = 1.05702; -const PINCH_ZOOM_OUT_FACTOR = 0.946058; const RENAME_NODE_NAME = 'Something else'; const RENAME_NODE_NAME2 = 'Something different'; @@ -41,27 +47,52 @@ describe('Canvas Node Manipulation and Navigation', () => { NDVDialog.actions.close(); for (let i = 0; i < desiredOutputs; i++) { - WorkflowPage.getters.canvasNodePlusEndpointByName(SWITCH_NODE_NAME, i).click({ force: true }); + cy.ifCanvasVersion( + () => { + WorkflowPage.getters + .canvasNodePlusEndpointByName(SWITCH_NODE_NAME, i) + .click({ force: true }); + }, + () => { + getOutputPlusHandle(SWITCH_NODE_NAME).eq(0).click(); + }, + ); WorkflowPage.getters.nodeCreatorSearchBar().should('be.visible'); WorkflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, false); WorkflowPage.actions.zoomToFit(); } WorkflowPage.getters.nodeViewBackground().click({ force: true }); - WorkflowPage.getters.canvasNodePlusEndpointByName(`${EDIT_FIELDS_SET_NODE_NAME}3`).click(); + cy.ifCanvasVersion( + () => { + WorkflowPage.getters.canvasNodePlusEndpointByName(`${EDIT_FIELDS_SET_NODE_NAME}3`).click(); + }, + () => { + getOutputPlusHandle(`${EDIT_FIELDS_SET_NODE_NAME}3`).click(); + }, + ); WorkflowPage.actions.addNodeToCanvas(SWITCH_NODE_NAME, false); WorkflowPage.actions.saveWorkflowOnButtonClick(); cy.reload(); cy.waitForLoad(); // Make sure outputless switch was connected correctly - WorkflowPage.getters - .getConnectionBetweenNodes(`${EDIT_FIELDS_SET_NODE_NAME}3`, `${SWITCH_NODE_NAME}1`) - .should('exist'); + cy.ifCanvasVersion( + () => { + WorkflowPage.getters + .getConnectionBetweenNodes(`${EDIT_FIELDS_SET_NODE_NAME}3`, `${SWITCH_NODE_NAME}1`) + .should('exist'); + }, + () => { + getConnectionBySourceAndTarget( + `${EDIT_FIELDS_SET_NODE_NAME}3`, + `${SWITCH_NODE_NAME}1`, + ).should('exist'); + }, + ); // Make sure all connections are there after reload for (let i = 0; i < desiredOutputs; i++) { const setName = `${EDIT_FIELDS_SET_NODE_NAME}${i > 0 ? i : ''}`; - WorkflowPage.getters - .getConnectionBetweenNodes(`${SWITCH_NODE_NAME}`, setName) - .should('exist'); + + getConnectionBySourceAndTarget(`${SWITCH_NODE_NAME}`, setName).should('exist'); } }); @@ -84,14 +115,29 @@ describe('Canvas Node Manipulation and Navigation', () => { ); // Connect Set1 and Set2 to merge - cy.draganddrop( - WorkflowPage.getters.getEndpointSelector('plus', EDIT_FIELDS_SET_NODE_NAME), - WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 0), - ); - cy.draganddrop( - WorkflowPage.getters.getEndpointSelector('plus', `${EDIT_FIELDS_SET_NODE_NAME}1`), - WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 1), + cy.ifCanvasVersion( + () => { + cy.draganddrop( + WorkflowPage.getters.getEndpointSelector('plus', EDIT_FIELDS_SET_NODE_NAME), + WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 0), + ); + cy.draganddrop( + WorkflowPage.getters.getEndpointSelector('plus', `${EDIT_FIELDS_SET_NODE_NAME}1`), + WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 1), + ); + }, + () => { + cy.draganddrop( + WorkflowPage.getters.getEndpointSelector('output', EDIT_FIELDS_SET_NODE_NAME), + WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 0), + ); + cy.draganddrop( + WorkflowPage.getters.getEndpointSelector('output', `${EDIT_FIELDS_SET_NODE_NAME}1`), + WorkflowPage.getters.getEndpointSelector('input', MERGE_NODE_NAME, 1), + ); + }, ); + const checkConnections = () => { WorkflowPage.getters .getConnectionBetweenNodes( @@ -117,10 +163,22 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.executeWorkflow(); WorkflowPage.getters.stopExecutionButton().should('not.exist'); + // Make sure all connections are there after save & reload + WorkflowPage.actions.saveWorkflowOnButtonClick(); + cy.reload(); + cy.waitForLoad(); + checkConnections(); + + WorkflowPage.actions.executeWorkflow(); + WorkflowPage.getters.stopExecutionButton().should('not.exist'); + // If the merged set nodes are connected and executed correctly, there should be 2 items in the output of merge node cy.ifCanvasVersion( () => cy.get('[data-label="2 items"]').should('be.visible'), - () => cy.getByTestId('canvas-node-output-handle').contains('2 items').should('be.visible'), + () => + getConnectionLabelBySourceAndTarget(`${EDIT_FIELDS_SET_NODE_NAME}1`, MERGE_NODE_NAME) + .contains('2 items') + .should('be.visible'), ); }); @@ -144,7 +202,10 @@ describe('Canvas Node Manipulation and Navigation', () => { cy.ifCanvasVersion( () => cy.get('.plus-draggable-endpoint').should('have.class', 'ep-success'), - () => cy.getByTestId('canvas-handle-plus').should('have.attr', 'data-plus-type', 'success'), + () => + cy + .getByTestId('canvas-handle-plus-wrapper') + .should('have.attr', 'data-plus-type', 'success'), ); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); @@ -212,8 +273,8 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); cy.wait(500); WorkflowPage.actions.selectAllFromContextMenu(); - WorkflowPage.actions.openContextMenu(); - WorkflowPage.actions.contextMenuAction('delete'); + openContextMenu(); + clickContextMenuAction('delete'); WorkflowPage.getters.canvasNodes().should('have.length', 0); }); @@ -228,41 +289,43 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); cy.wait(500); WorkflowPage.actions.selectAllFromContextMenu(); - WorkflowPage.actions.openContextMenu(); - WorkflowPage.actions.contextMenuAction('delete'); + openContextMenu(); + clickContextMenuAction('delete'); WorkflowPage.getters.canvasNodes().should('have.length', 0); }); - // FIXME: Canvas V2: Figure out how to test moving of the node it('should move node', () => { WorkflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.addNodeToCanvas(CODE_NODE_NAME); WorkflowPage.actions.zoomToFit(); - WorkflowPage.getters - .canvasNodes() + + getCanvasNodes() .last() .then(($node) => { - const { left, top } = $node.position(); + const { x: x1, y: y1 } = $node[0].getBoundingClientRect(); - if (isCanvasV2()) { - cy.drag('.vue-flow__node', [300, 300], { - realMouse: true, - }); - } else { - cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { - clickToFinish: true, - }); - } + cy.ifCanvasVersion( + () => { + cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 150], { + clickToFinish: true, + }); + }, + () => { + cy.drag(getCanvasNodes().last(), [50, 150], { + realMouse: true, + abs: true, + }); + }, + ); - WorkflowPage.getters - .canvasNodes() + getCanvasNodes() .last() .then(($node) => { - const { left: newLeft, top: newTop } = $node.position(); - expect(newLeft).to.be.greaterThan(left); - expect(newTop).to.be.greaterThan(top); + const { x: x2, y: y2 } = $node[0].getBoundingClientRect(); + expect(x2).to.be.greaterThan(x1); + expect(y2).to.be.greaterThan(y1); }); }); }); @@ -304,26 +367,6 @@ describe('Canvas Node Manipulation and Navigation', () => { zoomAndCheck('zoomOut', ZOOM_OUT_X2_FACTOR); }); - it('should zoom using scroll or pinch gesture', () => { - WorkflowPage.actions.pinchToZoom(1, 'zoomIn'); - - // V2 Canvas is using the same zoom factor for both pinch and scroll - cy.ifCanvasVersion( - () => checkZoomLevel(PINCH_ZOOM_IN_FACTOR), - () => checkZoomLevel(ZOOM_IN_X1_FACTOR), - ); - - WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); - checkZoomLevel(1); // Zoom in 1x + Zoom out 1x should reset to default (=1) - - WorkflowPage.actions.pinchToZoom(1, 'zoomOut'); - - cy.ifCanvasVersion( - () => checkZoomLevel(PINCH_ZOOM_OUT_FACTOR), - () => checkZoomLevel(ZOOM_OUT_X1_FACTOR), - ); - }); - it('should reset zoom', () => { WorkflowPage.getters.resetZoomButton().should('not.exist'); WorkflowPage.getters.zoomInButton().click(); @@ -369,7 +412,7 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.hitDisableNodeShortcut(); WorkflowPage.getters.disabledNodes().should('have.length', 0); WorkflowPage.actions.deselectAll(); - WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); + getCanvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); WorkflowPage.actions.hitDisableNodeShortcut(); WorkflowPage.getters.disabledNodes().should('have.length', 1); WorkflowPage.actions.hitSelectAll(); @@ -378,19 +421,19 @@ describe('Canvas Node Manipulation and Navigation', () => { // Context menu WorkflowPage.actions.hitSelectAll(); - WorkflowPage.actions.openContextMenu(); + openContextMenu(); WorkflowPage.actions.contextMenuAction('toggle_activation'); WorkflowPage.getters.disabledNodes().should('have.length', 0); - WorkflowPage.actions.openContextMenu(); + openContextMenu(); WorkflowPage.actions.contextMenuAction('toggle_activation'); WorkflowPage.getters.disabledNodes().should('have.length', 2); WorkflowPage.actions.deselectAll(); WorkflowPage.getters.canvasNodeByName(MANUAL_TRIGGER_NODE_DISPLAY_NAME).click(); - WorkflowPage.actions.openContextMenu(); + openContextMenu(); WorkflowPage.actions.contextMenuAction('toggle_activation'); WorkflowPage.getters.disabledNodes().should('have.length', 1); WorkflowPage.actions.hitSelectAll(); - WorkflowPage.actions.openContextMenu(); + openContextMenu(); WorkflowPage.actions.contextMenuAction('toggle_activation'); WorkflowPage.getters.disabledNodes().should('have.length', 2); }); @@ -466,7 +509,7 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.getters.canvasNodes().should('have.length', 2); WorkflowPage.getters.nodeConnections().should('have.length', 1); }); - // FIXME: Canvas V2: Credentials should show issue on the first open + it('should remove unknown credentials on pasting workflow', () => { cy.fixture('workflow-with-unknown-credentials.json').then((data) => { cy.get('body').paste(JSON.stringify(data)); @@ -478,35 +521,4 @@ describe('Canvas Node Manipulation and Navigation', () => { NDVDialog.actions.close(); }); }); - - // FIXME: Canvas V2: Unknown nodes should still render connection endpoints - it('should render connections correctly if unkown nodes are present', () => { - const unknownNodeName = 'Unknown node'; - cy.createFixtureWorkflow('workflow-with-unknown-nodes.json', 'Unknown nodes'); - - WorkflowPage.getters.canvasNodeByName(`${unknownNodeName} 1`).should('exist'); - WorkflowPage.getters.canvasNodeByName(`${unknownNodeName} 2`).should('exist'); - WorkflowPage.actions.zoomToFit(); - - cy.draganddrop( - WorkflowPage.getters.getEndpointSelector('plus', `${unknownNodeName} 1`), - WorkflowPage.getters.getEndpointSelector('input', EDIT_FIELDS_SET_NODE_NAME), - ); - - cy.draganddrop( - WorkflowPage.getters.getEndpointSelector('plus', `${unknownNodeName} 2`), - WorkflowPage.getters.getEndpointSelector('input', `${EDIT_FIELDS_SET_NODE_NAME}1`), - ); - - WorkflowPage.actions.executeWorkflow(); - cy.contains('Unrecognized node type').should('be.visible'); - - WorkflowPage.actions.deselectAll(); - WorkflowPage.actions.deleteNodeFromContextMenu(`${unknownNodeName} 1`); - WorkflowPage.actions.deleteNodeFromContextMenu(`${unknownNodeName} 2`); - - WorkflowPage.actions.executeWorkflow(); - - cy.contains('Unrecognized node type').should('not.exist'); - }); }); diff --git a/cypress/e2e/13-pinning.cy.ts b/cypress/e2e/13-pinning.cy.ts index 2d3351f8aa..800f9e417a 100644 --- a/cypress/e2e/13-pinning.cy.ts +++ b/cypress/e2e/13-pinning.cy.ts @@ -1,6 +1,3 @@ -import { nanoid } from 'nanoid'; - -import { simpleWebhookCall, waitForWebhook } from './16-webhook-node.cy'; import { HTTP_REQUEST_NODE_NAME, MANUAL_TRIGGER_NODE_NAME, @@ -109,36 +106,6 @@ describe('Data pinning', () => { ndv.getters.outputTbodyCell(1, 0).should('include.text', 1); }); - it('Should be able to pin data from canvas (context menu or shortcut)', () => { - workflowPage.actions.addInitialNodeToCanvas('Schedule Trigger'); - workflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME); - workflowPage.actions.openContextMenu(EDIT_FIELDS_SET_NODE_NAME, { method: 'overflow-button' }); - workflowPage.getters - .contextMenuAction('toggle_pin') - .parent() - .should('have.class', 'is-disabled'); - - cy.get('body').type('{esc}'); - - // Unpin using context menu - workflowPage.actions.openNode(EDIT_FIELDS_SET_NODE_NAME); - ndv.actions.setPinnedData([{ test: 1 }]); - ndv.actions.close(); - workflowPage.actions.pinNode(EDIT_FIELDS_SET_NODE_NAME); - workflowPage.actions.openNode(EDIT_FIELDS_SET_NODE_NAME); - ndv.getters.nodeOutputHint().should('exist'); - ndv.actions.close(); - - // Unpin using shortcut - workflowPage.actions.openNode(EDIT_FIELDS_SET_NODE_NAME); - ndv.actions.setPinnedData([{ test: 1 }]); - ndv.actions.close(); - workflowPage.getters.canvasNodeByName(EDIT_FIELDS_SET_NODE_NAME).click(); - workflowPage.actions.hitPinNodeShortcut(); - workflowPage.actions.openNode(EDIT_FIELDS_SET_NODE_NAME); - ndv.getters.nodeOutputHint().should('exist'); - }); - it('Should show an error when maximum pin data size is exceeded', () => { workflowPage.actions.addInitialNodeToCanvas('Schedule Trigger'); workflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, true, true); @@ -217,32 +184,6 @@ describe('Data pinning', () => { ); }); - it('should show pinned data tooltip', () => { - const { callEndpoint } = simpleWebhookCall({ - method: 'GET', - webhookPath: nanoid(), - executeNow: false, - }); - - ndv.actions.close(); - workflowPage.actions.executeWorkflow(); - cy.wait(waitForWebhook); - - // hide other visible popper on workflow execute button - workflowPage.getters.canvasNodes().eq(0).click(); - - callEndpoint((response) => { - expect(response.status).to.eq(200); - getVisiblePopper().should('have.length', 1); - getVisiblePopper() - .eq(0) - .should( - 'have.text', - 'You can pin this output instead of waiting for a test event. Open node to do so.', - ); - }); - }); - it('should not show pinned data tooltip', () => { cy.createFixtureWorkflow('Pinned_webhook_node.json', 'Test'); workflowPage.actions.executeWorkflow(); diff --git a/cypress/e2e/16-form-trigger-node.cy.ts b/cypress/e2e/16-form-trigger-node.cy.ts index ed901107ea..033753bc5c 100644 --- a/cypress/e2e/16-form-trigger-node.cy.ts +++ b/cypress/e2e/16-form-trigger-node.cy.ts @@ -62,12 +62,14 @@ describe('n8n Form Trigger', () => { getVisibleSelect().contains('Dropdown').click(); cy.contains('button', 'Add Field Option').click(); cy.contains('label', 'Field Options') + .parent() .parent() .nextAll() .find('[data-test-id="parameter-input-field"]') .eq(0) .type('Option 1'); cy.contains('label', 'Field Options') + .parent() .parent() .nextAll() .find('[data-test-id="parameter-input-field"]') diff --git a/cypress/e2e/16-webhook-node.cy.ts b/cypress/e2e/16-webhook-node.cy.ts index 3d6c1049a2..e0892a4a0b 100644 --- a/cypress/e2e/16-webhook-node.cy.ts +++ b/cypress/e2e/16-webhook-node.cy.ts @@ -1,5 +1,6 @@ import { nanoid } from 'nanoid'; +import { simpleWebhookCall, waitForWebhook } from '../composables/webhooks'; import { BACKEND_BASE_URL, EDIT_FIELDS_SET_NODE_NAME } from '../constants'; import { WorkflowPage, NDV, CredentialsModal } from '../pages'; import { cowBase64 } from '../support/binaryTestFiles'; @@ -9,81 +10,6 @@ const workflowPage = new WorkflowPage(); const ndv = new NDV(); const credentialsModal = new CredentialsModal(); -export const waitForWebhook = 500; - -interface SimpleWebhookCallOptions { - method: string; - webhookPath: string; - responseCode?: number; - respondWith?: string; - executeNow?: boolean; - responseData?: string; - authentication?: string; -} - -export const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { - const { - authentication, - method, - webhookPath, - responseCode, - respondWith, - responseData, - executeNow = true, - } = options; - - workflowPage.actions.addInitialNodeToCanvas('Webhook'); - workflowPage.actions.openNode('Webhook'); - - cy.getByTestId('parameter-input-httpMethod').click(); - getVisibleSelect().find('.option-headline').contains(method).click(); - cy.getByTestId('parameter-input-path') - .find('.parameter-input') - .find('input') - .clear() - .type(webhookPath); - - if (authentication) { - cy.getByTestId('parameter-input-authentication').click(); - getVisibleSelect().find('.option-headline').contains(authentication).click(); - } - - if (responseCode) { - cy.get('.param-options').click(); - getVisibleSelect().contains('Response Code').click(); - cy.get('.parameter-item-wrapper > .parameter-input-list-wrapper').children().click(); - getVisibleSelect().contains('201').click(); - } - - if (respondWith) { - cy.getByTestId('parameter-input-responseMode').click(); - getVisibleSelect().find('.option-headline').contains(respondWith).click(); - } - - if (responseData) { - cy.getByTestId('parameter-input-responseData').click(); - getVisibleSelect().find('.option-headline').contains(responseData).click(); - } - - const callEndpoint = (cb: (response: Cypress.Response) => void) => { - cy.request(method, `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then(cb); - }; - - if (executeNow) { - ndv.actions.execute(); - cy.wait(waitForWebhook); - - callEndpoint((response) => { - expect(response.status).to.eq(200); - ndv.getters.outputPanel().contains('headers'); - }); - } - - return { - callEndpoint, - }; -}; - describe('Webhook Trigger node', () => { beforeEach(() => { workflowPage.actions.visit(); @@ -250,7 +176,7 @@ describe('Webhook Trigger node', () => { }); // add credentials workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); credentialsModal.actions.fillCredentialsForm(); @@ -293,7 +219,7 @@ describe('Webhook Trigger node', () => { }); // add credentials workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); credentialsModal.actions.fillCredentialsForm(); diff --git a/cypress/e2e/17-sharing.cy.ts b/cypress/e2e/17-sharing.cy.ts index 30a990fb28..60eb474b07 100644 --- a/cypress/e2e/17-sharing.cy.ts +++ b/cypress/e2e/17-sharing.cy.ts @@ -297,10 +297,9 @@ describe('Credential Usage in Cross Shared Workflows', () => { workflowsPage.actions.createWorkflowFromCard(); workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); - // Only the credential in this project (+ the 'Create new' option) should - // be in the dropdown + // Only the credential in this project should be in the dropdown workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.length', 2); + getVisibleSelect().find('li').should('have.length', 1); }); it('should only show credentials in their personal project for members', () => { @@ -325,10 +324,9 @@ describe('Credential Usage in Cross Shared Workflows', () => { workflowsPage.actions.createWorkflowFromCard(); workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); - // Only the own credential the shared one (+ the 'Create new' option) - // should be in the dropdown + // Only the own credential the shared one should be in the dropdown workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.length', 3); + getVisibleSelect().find('li').should('have.length', 2); }); it('should only show credentials in their personal project for members if the workflow was shared with them', () => { @@ -355,10 +353,9 @@ describe('Credential Usage in Cross Shared Workflows', () => { workflowsPage.getters.workflowCardContent(workflowName).click(); workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); - // Only the own credential the shared one (+ the 'Create new' option) - // should be in the dropdown + // Only the own credential the shared one should be in the dropdown workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.length', 2); + getVisibleSelect().find('li').should('have.length', 1); }); it("should show all credentials from all personal projects the workflow's been shared into for the global owner", () => { @@ -400,10 +397,9 @@ describe('Credential Usage in Cross Shared Workflows', () => { workflowsPage.getters.workflowCardContent(workflowName).click(); workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); - // Only the personal credentials of the workflow owner and the global owner - // should show up. + // Only the personal credentials of the workflow owner and the global owner should show up. workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.length', 4); + getVisibleSelect().find('li').should('have.length', 3); }); it('should show all personal credentials if the global owner owns the workflow', () => { @@ -421,6 +417,6 @@ describe('Credential Usage in Cross Shared Workflows', () => { // Show all personal credentials workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.have.length', 2); + getVisibleSelect().find('li').should('have.have.length', 1); }); }); diff --git a/cypress/e2e/1858-PAY-can-use-context-menu.ts b/cypress/e2e/1858-PAY-can-use-context-menu.ts deleted file mode 100644 index 6727df4166..0000000000 --- a/cypress/e2e/1858-PAY-can-use-context-menu.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; - -const WorkflowPage = new WorkflowPageClass(); - -describe('PAY-1858 context menu', () => { - it('can use context menu on saved workflow', () => { - WorkflowPage.actions.visit(); - cy.createFixtureWorkflow('Test_workflow_filter.json', 'test'); - - WorkflowPage.getters.canvasNodes().should('have.length', 5); - WorkflowPage.actions.deleteNodeFromContextMenu('Then'); - WorkflowPage.getters.canvasNodes().should('have.length', 4); - - WorkflowPage.actions.hitSaveWorkflow(); - - cy.reload(); - WorkflowPage.getters.canvasNodes().should('have.length', 4); - WorkflowPage.actions.deleteNodeFromContextMenu('Code'); - WorkflowPage.getters.canvasNodes().should('have.length', 3); - }); -}); diff --git a/cypress/e2e/19-execution.cy.ts b/cypress/e2e/19-execution.cy.ts index 5be2399253..7563cc4827 100644 --- a/cypress/e2e/19-execution.cy.ts +++ b/cypress/e2e/19-execution.cy.ts @@ -1,3 +1,11 @@ +import { clickGetBackToCanvas, getNdvContainer, getOutputTableRow } from '../composables/ndv'; +import { + clickExecuteWorkflowButton, + getExecuteWorkflowButton, + getNodeByName, + getZoomToFitButton, + openNode, +} from '../composables/workflow'; import { SCHEDULE_TRIGGER_NODE_NAME, EDIT_FIELDS_SET_NODE_NAME } from '../constants'; import { NDV, WorkflowExecutionsTab, WorkflowPage as WorkflowPageClass } from '../pages'; import { clearNotifications, errorToast, successToast } from '../pages/notifications'; @@ -214,89 +222,37 @@ describe('Execution', () => { workflowPage.getters.clearExecutionDataButton().should('not.exist'); }); - // FIXME: Canvas V2: Webhook should show waiting state but it doesn't - it('should test webhook workflow stop', () => { - cy.createFixtureWorkflow('Webhook_wait_set.json'); + it('should test workflow with specific trigger node', () => { + cy.createFixtureWorkflow('Two_schedule_triggers.json'); - // Check workflow buttons - workflowPage.getters.executeWorkflowButton().should('be.visible'); - workflowPage.getters.clearExecutionDataButton().should('not.exist'); - workflowPage.getters.stopExecutionButton().should('not.exist'); - workflowPage.getters.stopExecutionWaitingForWebhookButton().should('not.exist'); + getZoomToFitButton().click(); + getExecuteWorkflowButton('Trigger A').should('not.be.visible'); + getExecuteWorkflowButton('Trigger B').should('not.be.visible'); - // Execute the workflow - workflowPage.getters.zoomToFitButton().click(); - workflowPage.getters.executeWorkflowButton().click(); + // Execute the workflow from trigger A + getNodeByName('Trigger A').realHover(); + getExecuteWorkflowButton('Trigger A').should('be.visible'); + getExecuteWorkflowButton('Trigger B').should('not.be.visible'); + clickExecuteWorkflowButton('Trigger A'); - // Check workflow buttons - workflowPage.getters.executeWorkflowButton().get('.n8n-spinner').should('be.visible'); - workflowPage.getters.clearExecutionDataButton().should('not.exist'); - workflowPage.getters.stopExecutionButton().should('not.exist'); - workflowPage.getters.stopExecutionWaitingForWebhookButton().should('be.visible'); + // Check the output + successToast().contains('Workflow executed successfully'); + openNode('Edit Fields'); + getOutputTableRow(1).should('include.text', 'Trigger A'); - workflowPage.getters.canvasNodes().first().dblclick(); + clickGetBackToCanvas(); + getNdvContainer().should('not.be.visible'); - ndv.getters.copyInput().click(); + // Execute the workflow from trigger B + getNodeByName('Trigger B').realHover(); + getExecuteWorkflowButton('Trigger A').should('not.be.visible'); + getExecuteWorkflowButton('Trigger B').should('be.visible'); + clickExecuteWorkflowButton('Trigger B'); - cy.grantBrowserPermissions('clipboardReadWrite', 'clipboardSanitizedWrite'); - - ndv.getters.backToCanvas().click(); - - cy.readClipboard().then((url) => { - cy.request({ - method: 'GET', - url, - }).then((resp) => { - expect(resp.status).to.eq(200); - }); - }); - - successToast().should('be.visible'); - clearNotifications(); - - workflowPage.getters.stopExecutionButton().click(); - // Check canvas nodes after 1st step (workflow passed the manual trigger node - workflowPage.getters - .canvasNodeByName('Webhook') - .within(() => cy.get('.fa-check')) - .should('exist'); - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-check').should('not.exist')); - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-sync-alt')) - .should('exist'); - workflowPage.getters - .canvasNodeByName('Set') - .within(() => cy.get('.fa-check').should('not.exist')); - - // Check canvas nodes after workflow stopped - workflowPage.getters - .canvasNodeByName('Webhook') - .within(() => cy.get('.fa-check')) - .should('exist'); - - if (isCanvasV2()) { - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-sync-alt').should('not.exist')); - } else { - workflowPage.getters - .canvasNodeByName('Wait') - .within(() => cy.get('.fa-sync-alt').should('not.be.visible')); - } - - workflowPage.getters - .canvasNodeByName('Set') - .within(() => cy.get('.fa-check').should('not.exist')); - - successToast().should('be.visible'); - - // Clear execution data - workflowPage.getters.clearExecutionDataButton().should('be.visible'); - workflowPage.getters.clearExecutionDataButton().click(); - workflowPage.getters.clearExecutionDataButton().should('not.exist'); + // Check the output + successToast().contains('Workflow executed successfully'); + openNode('Edit Fields'); + getOutputTableRow(1).should('include.text', 'Trigger B'); }); describe('execution preview', () => { @@ -312,8 +268,11 @@ describe('Execution', () => { }); }); - // FIXME: Canvas V2: Missing pinned states for `edge-label-wrapper` - describe('connections should be colored differently for pinned data', () => { + /** + * @TODO New Canvas: Different classes for pinned states on edges and nodes + */ + // eslint-disable-next-line n8n-local-rules/no-skipped-tests + describe.skip('connections should be colored differently for pinned data', () => { beforeEach(() => { cy.createFixtureWorkflow('Schedule_pinned.json'); workflowPage.actions.deselectAll(); @@ -634,45 +593,4 @@ describe('Execution', () => { errorToast().should('contain', 'Problem in node ‘Telegram‘'); }); - - it('should not show pinned data in production execution', () => { - cy.createFixtureWorkflow('Execution-pinned-data-check.json'); - - workflowPage.getters.zoomToFitButton().click(); - cy.intercept('PATCH', '/rest/workflows/*').as('workflowActivate'); - workflowPage.getters.activatorSwitch().click(); - - cy.wait('@workflowActivate'); - cy.get('body').type('{esc}'); - workflowPage.actions.openNode('Webhook'); - - cy.contains('label', 'Production URL').should('be.visible').click(); - cy.grantBrowserPermissions('clipboardReadWrite', 'clipboardSanitizedWrite'); - cy.get('.webhook-url').click(); - ndv.getters.backToCanvas().click(); - - cy.readClipboard().then((url) => { - cy.request({ - method: 'GET', - url, - }).then((resp) => { - expect(resp.status).to.eq(200); - }); - }); - - cy.intercept('GET', '/rest/executions/*').as('getExecution'); - executionsTab.actions.switchToExecutionsTab(); - - cy.wait('@getExecution'); - executionsTab.getters - .workflowExecutionPreviewIframe() - .should('be.visible') - .its('0.contentDocument.body') - .should('not.be.empty') - - .then(cy.wrap) - .find('.connection-run-items-label') - .filter(':contains("5 items")') - .should('have.length', 2); - }); }); diff --git a/cypress/e2e/2-credentials.cy.ts b/cypress/e2e/2-credentials.cy.ts index cda01c71a3..260d5f63a0 100644 --- a/cypress/e2e/2-credentials.cy.ts +++ b/cypress/e2e/2-credentials.cy.ts @@ -31,7 +31,7 @@ function createNotionCredential() { workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME); workflowPage.actions.openNode(NOTION_NODE_NAME); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.actions.fillCredentialsForm(); cy.get('body').type('{esc}'); workflowPage.actions.deleteNode(NOTION_NODE_NAME); @@ -79,7 +79,7 @@ describe('Credentials', () => { workflowPage.getters.canvasNodes().last().click(); cy.get('body').type('{enter}'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); credentialsModal.getters.credentialAuthTypeRadioButtons().should('have.length', 2); credentialsModal.getters.credentialAuthTypeRadioButtons().first().click(); @@ -99,7 +99,7 @@ describe('Credentials', () => { cy.get('body').type('{enter}'); workflowPage.getters.nodeCredentialsSelect().click(); // Add oAuth credentials - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); credentialsModal.getters.credentialAuthTypeRadioButtons().should('have.length', 2); credentialsModal.getters.credentialAuthTypeRadioButtons().first().click(); @@ -107,14 +107,13 @@ describe('Credentials', () => { cy.get('.el-message-box').find('button').contains('Close').click(); workflowPage.getters.nodeCredentialsSelect().click(); // Add Service account credentials - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); credentialsModal.getters.credentialAuthTypeRadioButtons().should('have.length', 2); credentialsModal.getters.credentialAuthTypeRadioButtons().last().click(); credentialsModal.actions.fillCredentialsForm(); - // Both (+ the 'Create new' option) should be in the dropdown workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').should('have.length.greaterThan', 2); + getVisibleSelect().find('li').should('have.length', 3); }); it('should correctly render required and optional credentials', () => { @@ -130,13 +129,13 @@ describe('Credentials', () => { workflowPage.getters.nodeCredentialsSelect().should('have.length', 2); workflowPage.getters.nodeCredentialsSelect().first().click(); - getVisibleSelect().find('li').contains('Create New Credential').click(); + workflowPage.getters.nodeCredentialsCreateOption().first().click(); // This one should show auth type selector credentialsModal.getters.credentialAuthTypeRadioButtons().should('have.length', 2); cy.get('body').type('{esc}'); workflowPage.getters.nodeCredentialsSelect().last().click(); - getVisibleSelect().find('li').contains('Create New Credential').click(); + workflowPage.getters.nodeCredentialsCreateOption().last().click(); // This one should not show auth type selector credentialsModal.getters.credentialsAuthTypeSelector().should('not.exist'); }); @@ -148,7 +147,7 @@ describe('Credentials', () => { workflowPage.getters.canvasNodes().last().click(); cy.get('body').type('{enter}'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialsAuthTypeSelector().should('not.exist'); credentialsModal.actions.fillCredentialsForm(); workflowPage.getters @@ -164,7 +163,7 @@ describe('Credentials', () => { workflowPage.getters.canvasNodes().last().click(); cy.get('body').type('{enter}'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.actions.fillCredentialsForm(); workflowPage.getters .nodeCredentialsSelect() @@ -189,7 +188,7 @@ describe('Credentials', () => { workflowPage.getters.canvasNodes().last().click(); cy.get('body').type('{enter}'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.actions.fillCredentialsForm(); workflowPage.getters.nodeCredentialsEditButton().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); @@ -232,7 +231,7 @@ describe('Credentials', () => { cy.getByTestId('credential-select').click(); cy.contains('Adalo API').click(); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.actions.fillCredentialsForm(); workflowPage.getters.nodeCredentialsEditButton().click(); credentialsModal.getters.credentialsEditModal().should('be.visible'); @@ -296,7 +295,7 @@ describe('Credentials', () => { workflowPage.getters.nodeCredentialsSelect().should('exist'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.actions.fillCredentialsForm(); workflowPage.getters .nodeCredentialsSelect() @@ -325,7 +324,7 @@ describe('Credentials', () => { workflowPage.actions.addNodeToCanvas('Slack', true, true, 'Get a channel'); workflowPage.getters.nodeCredentialsSelect().should('exist'); workflowPage.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialAuthTypeRadioButtons().first().click(); nodeDetailsView.getters.copyInput().should('not.exist'); }); diff --git a/cypress/e2e/21-community-nodes.cy.ts b/cypress/e2e/21-community-nodes.cy.ts index 17f82ec573..283c08d557 100644 --- a/cypress/e2e/21-community-nodes.cy.ts +++ b/cypress/e2e/21-community-nodes.cy.ts @@ -89,7 +89,7 @@ describe('Community and custom nodes in canvas', () => { workflowPage.actions.addNodeToCanvas('Manual'); workflowPage.actions.addNodeToCanvas('E2E Node with native n8n credential', true, true); workflowPage.getters.nodeCredentialsLabel().click(); - cy.contains('Create New Credential').click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.editCredentialModal().should('be.visible'); credentialsModal.getters.editCredentialModal().should('contain.text', 'Notion API'); }); @@ -98,7 +98,7 @@ describe('Community and custom nodes in canvas', () => { workflowPage.actions.addNodeToCanvas('Manual'); workflowPage.actions.addNodeToCanvas('E2E Node with custom credential', true, true); workflowPage.getters.nodeCredentialsLabel().click(); - cy.contains('Create New Credential').click(); + workflowPage.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.editCredentialModal().should('be.visible'); credentialsModal.getters.editCredentialModal().should('contain.text', 'Custom E2E Credential'); }); diff --git a/cypress/e2e/2106-ADO-pinned-data-execution-preview.cy.ts b/cypress/e2e/2106-ADO-pinned-data-execution-preview.cy.ts deleted file mode 100644 index e26a7acb82..0000000000 --- a/cypress/e2e/2106-ADO-pinned-data-execution-preview.cy.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { WorkflowExecutionsTab, WorkflowPage as WorkflowPageClass } from '../pages'; - -const workflowPage = new WorkflowPageClass(); -const executionsTab = new WorkflowExecutionsTab(); - -describe('ADO-2106 connections should be colored correctly for pinned data in executions preview', () => { - beforeEach(() => { - workflowPage.actions.visit(); - }); - - beforeEach(() => { - cy.createFixtureWorkflow('Webhook_set_pinned.json'); - workflowPage.actions.deselectAll(); - workflowPage.getters.zoomToFitButton().click(); - - workflowPage.getters.getConnectionBetweenNodes('Webhook', 'Set').should('have.class', 'pinned'); - }); - - it('should color connections for pinned data nodes for manual executions', () => { - workflowPage.actions.executeWorkflow(); - - executionsTab.actions.switchToExecutionsTab(); - - executionsTab.getters.successfulExecutionListItems().should('have.length', 1); - - executionsTab.getters - .workflowExecutionPreviewIframe() - .should('be.visible') - .its('0.contentDocument.body') - .should('not.be.empty') - - .then(cy.wrap) - .find('.jtk-connector[data-source-node="Webhook"][data-target-node="Set"]') - .should('have.class', 'success') - .should('have.class', 'has-run') - .should('have.class', 'pinned'); - }); -}); diff --git a/cypress/e2e/2111-ado-support-pinned-data-in-expressions.cy.ts b/cypress/e2e/2111-ado-support-pinned-data-in-expressions.cy.ts deleted file mode 100644 index 6d2da55b32..0000000000 --- a/cypress/e2e/2111-ado-support-pinned-data-in-expressions.cy.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { WorkflowPage, NDV } from '../pages'; - -const workflowPage = new WorkflowPage(); -const ndv = new NDV(); - -describe('ADO-2111 expressions should support pinned data', () => { - beforeEach(() => { - workflowPage.actions.visit(); - }); - - it('supports pinned data in expressions unexecuted and executed parent nodes', () => { - cy.createFixtureWorkflow('Test_workflow_pinned_data_in_expressions.json', 'Expressions'); - - // test previous node unexecuted - workflowPage.actions.openNode('NotPinnedWithExpressions'); - ndv.getters - .parameterExpressionPreview('value') - .eq(0) - .should('include.text', 'Joe\nJoe\nJoan\nJoan\nJoe\nJoan\n\nJoe\nJoan\n\nJoe'); - ndv.getters - .parameterExpressionPreview('value') - .eq(1) - .should('contain.text', '0,0\nJoe\n\nJoe\n\nJoe\n\nJoe\nJoe'); - - // test can resolve correctly based on item - ndv.actions.switchInputMode('Table'); - - ndv.getters.inputTableRow(2).realHover(); - cy.wait(50); - ndv.getters - .parameterExpressionPreview('value') - .eq(0) - .should('include.text', 'Joe\nJoe\nJoan\nJoan\nJoe\nJoan\n\nJoe\nJoan\n\nJoe'); - ndv.getters - .parameterExpressionPreview('value') - .eq(1) - .should('contain.text', '0,1\nJoan\n\nJoan\n\nJoan\n\nJoan\nJoan'); - - // test previous node executed - ndv.actions.execute(); - ndv.getters.inputTableRow(1).realHover(); - cy.wait(50); - - ndv.getters - .parameterExpressionPreview('value') - .eq(0) - .should('include.text', 'Joe\nJoe\nJoan\nJoan\nJoe\nJoan\n\nJoe\nJoan\n\nJoe'); - - ndv.getters - .parameterExpressionPreview('value') - .eq(1) - .should('contain.text', '0,0\nJoe\n\nJoe\n\nJoe\n\nJoe\nJoe'); - - ndv.getters.inputTableRow(2).realHover(); - cy.wait(50); - ndv.getters - .parameterExpressionPreview('value') - .eq(0) - .should('include.text', 'Joe\nJoe\nJoan\nJoan\nJoe\nJoan\n\nJoe\nJoan\n\nJoe'); - ndv.getters - .parameterExpressionPreview('value') - .eq(1) - .should('contain.text', '0,1\nJoan\n\nJoan\n\nJoan\n\nJoan\nJoan'); - - // check it resolved correctly on the backend - ndv.getters - .outputTbodyCell(1, 0) - .should('contain.text', 'Joe\\nJoe\\nJoan\\nJoan\\nJoe\\nJoan\\n\\nJoe\\nJoan\\n\\nJoe'); - - ndv.getters - .outputTbodyCell(2, 0) - .should('contain.text', 'Joe\\nJoe\\nJoan\\nJoan\\nJoe\\nJoan\\n\\nJoe\\nJoan\\n\\nJoe'); - - ndv.getters - .outputTbodyCell(1, 1) - .should('contain.text', '0,0\\nJoe\\n\\nJoe\\n\\nJoe\\n\\nJoe\\nJoe'); - - ndv.getters - .outputTbodyCell(2, 1) - .should('contain.text', '0,1\\nJoan\\n\\nJoan\\n\\nJoan\\n\\nJoan\\nJoan'); - }); - - it('resets expressions after node is unpinned', () => { - cy.createFixtureWorkflow('Test_workflow_pinned_data_in_expressions.json', 'Expressions'); - - // test previous node unexecuted - workflowPage.actions.openNode('NotPinnedWithExpressions'); - ndv.getters - .parameterExpressionPreview('value') - .eq(0) - .should('include.text', 'Joe\nJoe\nJoan\nJoan\nJoe\nJoan\n\nJoe\nJoan\n\nJoe'); - ndv.getters - .parameterExpressionPreview('value') - .eq(1) - .should('contain.text', '0,0\nJoe\n\nJoe\n\nJoe\n\nJoe\nJoe'); - - ndv.actions.close(); - - // unpin pinned node - workflowPage.getters - .canvasNodeByName('PinnedSet') - .eq(0) - .find('.node-pin-data-icon') - .should('exist'); - workflowPage.getters.canvasNodeByName('PinnedSet').eq(0).click(); - workflowPage.actions.hitPinNodeShortcut(); - workflowPage.getters - .canvasNodeByName('PinnedSet') - .eq(0) - .find('.node-pin-data-icon') - .should('not.exist'); - - workflowPage.actions.openNode('NotPinnedWithExpressions'); - ndv.getters.nodeParameters().find('parameter-expression-preview-value').should('not.exist'); - - ndv.getters.parameterInput('value').eq(0).click(); - ndv.getters - .inlineExpressionEditorOutput() - .should( - 'have.text', - '[Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute previous nodes for preview][Execute previous nodes for preview][undefined]', - ); - - // close open expression - ndv.getters.inputLabel().eq(0).click(); - - ndv.getters.parameterInput('value').eq(1).click(); - ndv.getters - .inlineExpressionEditorOutput() - .should( - 'have.text', - '0,0[Execute node ‘PinnedSet’ for preview][Execute node ‘PinnedSet’ for preview][Execute previous nodes for preview][Execute previous nodes for preview][Execute previous nodes for preview]', - ); - }); -}); diff --git a/cypress/e2e/23-variables.cy.ts b/cypress/e2e/23-variables.cy.ts index c481f25128..a0af3c09ac 100644 --- a/cypress/e2e/23-variables.cy.ts +++ b/cypress/e2e/23-variables.cy.ts @@ -65,8 +65,11 @@ describe('Variables', () => { const editingRow = variablesPage.getters.variablesEditableRows().eq(0); variablesPage.actions.setRowValue(editingRow, 'key', key); variablesPage.actions.setRowValue(editingRow, 'value', value); - editingRow.should('contain', 'This field may contain only letters'); - variablesPage.getters.editableRowSaveButton(editingRow).should('be.disabled'); + variablesPage.actions.saveRowEditing(editingRow); + variablesPage.getters + .variablesEditableRows() + .eq(0) + .should('contain', 'This field may contain only letters'); variablesPage.actions.cancelRowEditing(editingRow); variablesPage.getters.variablesRows().should('have.length', 3); diff --git a/cypress/e2e/24-ndv-paired-item.cy.ts b/cypress/e2e/24-ndv-paired-item.cy.ts index 49257a8a12..ccae14f6c9 100644 --- a/cypress/e2e/24-ndv-paired-item.cy.ts +++ b/cypress/e2e/24-ndv-paired-item.cy.ts @@ -118,6 +118,15 @@ describe('NDV', () => { ndv.actions.switchInputMode('Table'); ndv.actions.switchOutputMode('Table'); + // Start from linked state + ndv.getters.outputLinkRun().then(($el) => { + const classList = Array.from($el[0].classList); + if (!classList.includes('linked')) { + ndv.actions.toggleOutputRunLinking(); + ndv.getters.inputTbodyCell(1, 0).click(); // remove tooltip + } + }); + ndv.getters .inputRunSelector() .should('exist') @@ -243,38 +252,38 @@ describe('NDV', () => { // biome-ignore format: const PINNED_DATA = [ { - "id": "abc", - "historyId": "def", - "messages": [ + id: 'abc', + historyId: 'def', + messages: [ { - "id": "abc" - } - ] + id: 'abc', + }, + ], }, { - "id": "abc", - "historyId": "def", - "messages": [ + id: 'abc', + historyId: 'def', + messages: [ { - "id": "abc" + id: 'abc', }, { - "id": "abc" + id: 'abc', }, { - "id": "abc" - } - ] + id: 'abc', + }, + ], }, { - "id": "abc", - "historyId": "def", - "messages": [ + id: 'abc', + historyId: 'def', + messages: [ { - "id": "abc" - } - ] - } + id: 'abc', + }, + ], + }, ]; workflowPage.actions.openNode('Get thread details1'); ndv.actions.pastePinnedData(PINNED_DATA); diff --git a/cypress/e2e/25-stickies.cy.ts b/cypress/e2e/25-stickies.cy.ts index 14c176f17b..da8d6c2674 100644 --- a/cypress/e2e/25-stickies.cy.ts +++ b/cypress/e2e/25-stickies.cy.ts @@ -3,24 +3,6 @@ import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; const workflowPage = new WorkflowPageClass(); -function checkStickiesStyle( - top: number, - left: number, - height: number, - width: number, - zIndex?: number, -) { - workflowPage.getters.stickies().should(($el) => { - expect($el).to.have.css('top', `${top}px`); - expect($el).to.have.css('left', `${left}px`); - expect($el).to.have.css('height', `${height}px`); - expect($el).to.have.css('width', `${width}px`); - if (zIndex) { - expect($el).to.have.css('z-index', `${zIndex}`); - } - }); -} - describe('Canvas Actions', () => { beforeEach(() => { workflowPage.actions.visit(); @@ -51,191 +33,8 @@ describe('Canvas Actions', () => { .contains('Guide') .should('have.attr', 'href'); }); - - it('drags sticky around to top left corner', () => { - // used to caliberate move sticky function - addDefaultSticky(); - moveSticky({ top: 0, left: 0 }); - }); - - it('drags sticky around and position/size are saved correctly', () => { - addDefaultSticky(); - moveSticky({ top: 500, left: 500 }); - - workflowPage.actions.saveWorkflowOnButtonClick(); - cy.wait('@createWorkflow'); - - cy.reload(); - cy.waitForLoad(); - - stickyShouldBePositionedCorrectly({ top: 500, left: 500 }); - }); - - it('deletes sticky', () => { - workflowPage.actions.addSticky(); - workflowPage.getters.stickies().should('have.length', 1); - - workflowPage.actions.deleteSticky(); - - workflowPage.getters.stickies().should('have.length', 0); - }); - - it('edits sticky and updates content as markdown', () => { - workflowPage.actions.addSticky(); - - workflowPage.getters - .stickies() - .should('have.text', 'I’m a note\nDouble click to edit me. Guide\n'); - - workflowPage.getters.stickies().dblclick(); - workflowPage.actions.editSticky('# hello world \n ## text text'); - workflowPage.getters.stickies().find('h1').should('have.text', 'hello world'); - workflowPage.getters.stickies().find('h2').should('have.text', 'text text'); - }); - - it('expands/shrinks sticky from the right edge', () => { - addDefaultSticky(); - - moveSticky({ top: 200, left: 200 }); - - cy.drag('[data-test-id="sticky"] [data-dir="right"]', [100, 100]); - checkStickiesStyle(100, 20, 160, 346); - - cy.drag('[data-test-id="sticky"] [data-dir="right"]', [-50, -50]); - checkStickiesStyle(100, 20, 160, 302); - }); - - it('expands/shrinks sticky from the left edge', () => { - addDefaultSticky(); - - moveSticky({ left: 600, top: 200 }); - cy.drag('[data-test-id="sticky"] [data-dir="left"]', [100, 100]); - checkStickiesStyle(100, 510, 160, 150); - - cy.drag('[data-test-id="sticky"] [data-dir="left"]', [-50, -50]); - checkStickiesStyle(100, 466, 160, 194); - }); - - it('expands/shrinks sticky from the top edge', () => { - workflowPage.actions.addSticky(); - cy.drag('[data-test-id="sticky"]', [100, 100]); // move away from canvas button - checkStickiesStyle(300, 620, 160, 240); - - cy.drag('[data-test-id="sticky"] [data-dir="top"]', [100, 100]); - checkStickiesStyle(380, 620, 80, 240); - - cy.drag('[data-test-id="sticky"] [data-dir="top"]', [-50, -50]); - checkStickiesStyle(324, 620, 136, 240); - }); - - it('expands/shrinks sticky from the bottom edge', () => { - workflowPage.actions.addSticky(); - cy.drag('[data-test-id="sticky"]', [100, 100]); // move away from canvas button - checkStickiesStyle(300, 620, 160, 240); - - cy.drag('[data-test-id="sticky"] [data-dir="bottom"]', [100, 100]); - checkStickiesStyle(300, 620, 254, 240); - - cy.drag('[data-test-id="sticky"] [data-dir="bottom"]', [-50, -50]); - checkStickiesStyle(300, 620, 198, 240); - }); - - it('expands/shrinks sticky from the bottom right edge', () => { - workflowPage.actions.addSticky(); - cy.drag('[data-test-id="sticky"]', [-100, -100]); // move away from canvas button - checkStickiesStyle(100, 420, 160, 240); - - cy.drag('[data-test-id="sticky"] [data-dir="bottomRight"]', [100, 100]); - checkStickiesStyle(100, 420, 254, 346); - - cy.drag('[data-test-id="sticky"] [data-dir="bottomRight"]', [-50, -50]); - checkStickiesStyle(100, 420, 198, 302); - }); - - it('expands/shrinks sticky from the top right edge', () => { - addDefaultSticky(); - - cy.drag('[data-test-id="sticky"] [data-dir="topRight"]', [100, 100]); - checkStickiesStyle(360, 400, 80, 346); - - cy.drag('[data-test-id="sticky"] [data-dir="topRight"]', [-50, -50]); - checkStickiesStyle(304, 400, 136, 302); - }); - - it('expands/shrinks sticky from the top left edge, and reach min height/width', () => { - addDefaultSticky(); - - cy.drag('[data-test-id="sticky"] [data-dir="topLeft"]', [100, 100]); - checkStickiesStyle(360, 490, 80, 150); - - cy.drag('[data-test-id="sticky"] [data-dir="topLeft"]', [-150, -150]); - checkStickiesStyle(204, 346, 236, 294); - }); - - it('sets sticky behind node', () => { - workflowPage.actions.addInitialNodeToCanvas('Manual Trigger'); - addDefaultSticky(); - - cy.drag('[data-test-id="sticky"] [data-dir="topLeft"]', [-150, -150]); - checkStickiesStyle(124, 256, 316, 384, -121); - - workflowPage.getters - .canvasNodes() - .eq(0) - .should(($el) => { - expect($el).to.have.css('z-index', 'auto'); - }); - - workflowPage.actions.addSticky(); - workflowPage.getters - .stickies() - .eq(0) - .should(($el) => { - expect($el).to.have.css('z-index', '-121'); - }); - workflowPage.getters - .stickies() - .eq(1) - .should(($el) => { - expect($el).to.have.css('z-index', '-38'); - }); - - cy.drag('[data-test-id="sticky"] [data-dir="topLeft"]', [-200, -200], { index: 1 }); - workflowPage.getters - .stickies() - .eq(0) - .should(($el) => { - expect($el).to.have.css('z-index', '-121'); - }); - - workflowPage.getters - .stickies() - .eq(1) - .should(($el) => { - expect($el).to.have.css('z-index', '-158'); - }); - }); - - it('Empty sticky should not error when activating workflow', () => { - workflowPage.actions.addSticky(); - - workflowPage.getters.stickies().should('have.length', 1); - - workflowPage.getters.stickies().dblclick(); - - workflowPage.actions.clearSticky(); - - workflowPage.actions.addNodeToCanvas('Schedule Trigger'); - - workflowPage.actions.activateWorkflow(); - }); }); -type Position = { - top: number; - left: number; -}; - function shouldHaveOneSticky() { workflowPage.getters.stickies().should('have.length', 1); } @@ -263,17 +62,3 @@ function addDefaultSticky() { shouldHaveDefaultSize(); shouldBeInDefaultLocation(); } - -function stickyShouldBePositionedCorrectly(position: Position) { - const yOffset = -100; - const xOffset = -180; - workflowPage.getters.stickies().should(($el) => { - expect($el).to.have.css('top', `${yOffset + position.top}px`); - expect($el).to.have.css('left', `${xOffset + position.left}px`); - }); -} - -function moveSticky(target: Position) { - cy.drag('[data-test-id="sticky"]', [target.left, target.top], { abs: true }); - stickyShouldBePositionedCorrectly(target); -} diff --git a/cypress/e2e/30-editor-after-route-changes.cy.ts b/cypress/e2e/30-editor-after-route-changes.cy.ts index 307c4a9537..89c64e1156 100644 --- a/cypress/e2e/30-editor-after-route-changes.cy.ts +++ b/cypress/e2e/30-editor-after-route-changes.cy.ts @@ -1,86 +1,7 @@ import { getWorkflowHistoryCloseButton } from '../composables/workflow'; -import { - CODE_NODE_NAME, - EDIT_FIELDS_SET_NODE_NAME, - IF_NODE_NAME, - SCHEDULE_TRIGGER_NODE_NAME, -} from '../constants'; -import { WorkflowExecutionsTab, WorkflowPage as WorkflowPageClass } from '../pages'; +import { WorkflowPage as WorkflowPageClass } from '../pages'; const workflowPage = new WorkflowPageClass(); -const executionsTab = new WorkflowExecutionsTab(); - -const createNewWorkflowAndActivate = () => { - workflowPage.actions.visit(); - workflowPage.actions.addNodeToCanvas(SCHEDULE_TRIGGER_NODE_NAME); - workflowPage.actions.saveWorkflowOnButtonClick(); - workflowPage.actions.activateWorkflow(); - cy.get('.el-notification .el-notification--error').should('not.exist'); -}; - -const editWorkflowAndDeactivate = () => { - workflowPage.getters.canvasNodePlusEndpointByName(SCHEDULE_TRIGGER_NODE_NAME).click(); - workflowPage.getters.nodeCreatorSearchBar().should('be.visible'); - workflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, false); - cy.get('.jtk-connector').should('have.length', 1); - workflowPage.actions.saveWorkflowOnButtonClick(); - workflowPage.getters.activatorSwitch().click(); - workflowPage.actions.zoomToFit(); - cy.get('.el-notification .el-notification--error').should('not.exist'); -}; - -const editWorkflowMoreAndActivate = () => { - cy.drag(workflowPage.getters.getEndpointSelector('plus', EDIT_FIELDS_SET_NODE_NAME), [200, 200], { - realMouse: true, - }); - workflowPage.getters.nodeCreatorSearchBar().should('be.visible'); - - workflowPage.actions.addNodeToCanvas(CODE_NODE_NAME, false); - workflowPage.getters.nodeViewBackground().click(600, 200, { force: true }); - cy.get('.jtk-connector').should('have.length', 2); - workflowPage.actions.zoomToFit(); - workflowPage.actions.saveWorkflowOnButtonClick(); - - workflowPage.actions.addNodeToCanvas(IF_NODE_NAME); - workflowPage.getters.nodeViewBackground().click(600, 200, { force: true }); - cy.get('.jtk-connector').should('have.length', 2); - - const position = { - top: 0, - left: 0, - }; - workflowPage.getters - .canvasNodeByName(IF_NODE_NAME) - .click() - .then(($element) => { - position.top = $element.position().top; - position.left = $element.position().left; - }); - - cy.drag('[data-test-id="canvas-node"].jtk-drag-selected', [50, 200], { clickToFinish: true }); - workflowPage.getters - .canvasNodes() - .last() - .then(($element) => { - const finalPosition = { - top: $element.position().top, - left: $element.position().left, - }; - - expect(finalPosition.top).to.be.greaterThan(position.top); - expect(finalPosition.left).to.be.greaterThan(position.left); - }); - - cy.draganddrop( - workflowPage.getters.getEndpointSelector('output', CODE_NODE_NAME), - workflowPage.getters.getEndpointSelector('input', IF_NODE_NAME), - ); - cy.get('.jtk-connector').should('have.length', 3); - - workflowPage.actions.saveWorkflowOnButtonClick(); - workflowPage.getters.activatorSwitch().click(); - cy.get('.el-notification .el-notification--error').should('not.exist'); -}; const switchBetweenEditorAndHistory = () => { workflowPage.getters.workflowHistoryButton().click(); @@ -116,62 +37,6 @@ const zoomInAndCheckNodes = () => { workflowPage.getters.canvasNodes().last().should('not.be.visible'); }; -describe('Editor actions should work', () => { - beforeEach(() => { - cy.enableFeature('debugInEditor'); - cy.enableFeature('workflowHistory'); - cy.signinAsOwner(); - createNewWorkflowAndActivate(); - }); - - it('after switching between Editor and Executions', () => { - cy.intercept('GET', '/rest/executions?filter=*').as('getExecutions'); - - executionsTab.actions.switchToExecutionsTab(); - cy.wait(['@getExecutions']); - cy.wait(500); - executionsTab.actions.switchToEditorTab(); - editWorkflowAndDeactivate(); - editWorkflowMoreAndActivate(); - }); - - it('after switching between Editor and Debug', () => { - cy.intercept('GET', '/rest/executions?filter=*').as('getExecutions'); - cy.intercept('GET', '/rest/executions/*').as('getExecution'); - cy.intercept('POST', '/rest/workflows/**/run?**').as('postWorkflowRun'); - - editWorkflowAndDeactivate(); - workflowPage.actions.executeWorkflow(); - cy.wait(['@postWorkflowRun']); - - executionsTab.actions.switchToExecutionsTab(); - cy.wait(['@getExecutions']); - - executionsTab.getters.executionListItems().should('have.length', 1).first().click(); - cy.wait(['@getExecution']); - - executionsTab.getters.executionDebugButton().should('have.text', 'Copy to editor').click(); - editWorkflowMoreAndActivate(); - }); - - it('after switching between Editor and Workflow history', () => { - cy.intercept('GET', '/rest/workflow-history/workflow/*/version/*').as('getVersion'); - cy.intercept('GET', '/rest/workflow-history/workflow/*').as('getHistory'); - - editWorkflowAndDeactivate(); - workflowPage.getters.workflowHistoryButton().click(); - cy.wait(['@getHistory']); - cy.wait(['@getVersion']); - - cy.intercept('GET', '/rest/workflows/*').as('workflowGet'); - getWorkflowHistoryCloseButton().click(); - cy.wait(['@workflowGet']); - cy.wait(1000); - - editWorkflowMoreAndActivate(); - }); -}); - describe('Editor zoom should work after route changes', () => { beforeEach(() => { cy.enableFeature('debugInEditor'); diff --git a/cypress/e2e/30-langchain.cy.ts b/cypress/e2e/30-langchain.cy.ts index 96a03be961..2d0076bc11 100644 --- a/cypress/e2e/30-langchain.cy.ts +++ b/cypress/e2e/30-langchain.cy.ts @@ -28,7 +28,7 @@ import { clickGetBackToCanvas, getRunDataInfoCallout, getOutputPanelTable, - toggleParameterCheckboxInputByName, + checkParameterCheckboxInputByName, } from '../composables/ndv'; import { addLanguageModelNodeToParent, @@ -38,8 +38,6 @@ import { addToolNodeToParent, clickExecuteWorkflowButton, clickManualChatButton, - disableNode, - getExecuteWorkflowButton, navigateToNewWorkflowPage, getNodes, openNode, @@ -73,31 +71,10 @@ describe('Langchain Integration', () => { getManualChatModal().should('not.exist'); }); - it('should disable test workflow button', () => { - addNodeToCanvas('Schedule Trigger', true); - addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, true); - - clickGetBackToCanvas(); - - addNodeToCanvas(AGENT_NODE_NAME, true, true); - clickGetBackToCanvas(); - - addLanguageModelNodeToParent( - AI_LANGUAGE_MODEL_OPENAI_CHAT_MODEL_NODE_NAME, - AGENT_NODE_NAME, - true, - ); - clickGetBackToCanvas(); - - disableNode('Schedule Trigger'); - - getExecuteWorkflowButton().should('be.disabled'); - }); - it('should add nodes to all Agent node input types', () => { addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME, true); addNodeToCanvas(AGENT_NODE_NAME, true, true); - toggleParameterCheckboxInputByName('hasOutputParser'); + checkParameterCheckboxInputByName('hasOutputParser'); clickGetBackToCanvas(); addLanguageModelNodeToParent( @@ -368,58 +345,6 @@ describe('Langchain Integration', () => { getConnectionBySourceAndTarget(CHAT_TRIGGER_NODE_DISPLAY_NAME, AGENT_NODE_NAME).should('exist'); getNodes().should('have.length', 3); }); - it('should render runItems for sub-nodes and allow switching between them', () => { - const workflowPage = new WorkflowPage(); - const ndv = new NDV(); - - cy.visit(workflowPage.url); - cy.createFixtureWorkflow('In_memory_vector_store_fake_embeddings.json'); - workflowPage.actions.zoomToFit(); - - workflowPage.actions.executeNode('Populate VS'); - cy.get('[data-label="25 items"]').should('exist'); - - const assertInputOutputText = (text: string, assertion: 'exist' | 'not.exist') => { - ndv.getters.outputPanel().contains(text).should(assertion); - ndv.getters.inputPanel().contains(text).should(assertion); - }; - - workflowPage.actions.openNode('Character Text Splitter'); - ndv.getters.outputRunSelector().should('exist'); - ndv.getters.inputRunSelector().should('exist'); - ndv.getters.inputRunSelector().find('input').should('include.value', '3 of 3'); - ndv.getters.outputRunSelector().find('input').should('include.value', '3 of 3'); - assertInputOutputText('Kyiv', 'exist'); - assertInputOutputText('Berlin', 'not.exist'); - assertInputOutputText('Prague', 'not.exist'); - - ndv.actions.changeOutputRunSelector('2 of 3'); - assertInputOutputText('Berlin', 'exist'); - assertInputOutputText('Kyiv', 'not.exist'); - assertInputOutputText('Prague', 'not.exist'); - - ndv.actions.changeOutputRunSelector('1 of 3'); - assertInputOutputText('Prague', 'exist'); - assertInputOutputText('Berlin', 'not.exist'); - assertInputOutputText('Kyiv', 'not.exist'); - - ndv.actions.toggleInputRunLinking(); - ndv.actions.changeOutputRunSelector('2 of 3'); - ndv.getters.inputRunSelector().find('input').should('include.value', '1 of 3'); - ndv.getters.outputRunSelector().find('input').should('include.value', '2 of 3'); - ndv.getters.inputPanel().contains('Prague').should('exist'); - ndv.getters.inputPanel().contains('Berlin').should('not.exist'); - - ndv.getters.outputPanel().contains('Berlin').should('exist'); - ndv.getters.outputPanel().contains('Prague').should('not.exist'); - - ndv.actions.toggleInputRunLinking(); - ndv.getters.inputRunSelector().find('input').should('include.value', '1 of 3'); - ndv.getters.outputRunSelector().find('input').should('include.value', '1 of 3'); - assertInputOutputText('Prague', 'exist'); - assertInputOutputText('Berlin', 'not.exist'); - assertInputOutputText('Kyiv', 'not.exist'); - }); it('should show tool info notice if no existing tools were used during execution', () => { addNodeToCanvas(MANUAL_CHAT_TRIGGER_NODE_NAME, true); @@ -518,4 +443,29 @@ describe('Langchain Integration', () => { getRunDataInfoCallout().should('not.exist'); }); + + it('should execute up to Node 1 when using partial execution', () => { + const workflowPage = new WorkflowPage(); + const ndv = new NDV(); + + cy.visit(workflowPage.url); + cy.createFixtureWorkflow('Test_workflow_chat_partial_execution.json'); + workflowPage.actions.zoomToFit(); + + getManualChatModal().should('not.exist'); + openNode('Node 1'); + ndv.actions.execute(); + + getManualChatModal().should('exist'); + sendManualChatMessage('Test'); + + getManualChatMessages().should('contain', 'this_my_field_1'); + cy.getByTestId('refresh-session-button').click(); + cy.get('button').contains('Reset').click(); + getManualChatMessages().should('not.exist'); + + sendManualChatMessage('Another test'); + getManualChatMessages().should('contain', 'this_my_field_3'); + getManualChatMessages().should('contain', 'this_my_field_4'); + }); }); diff --git a/cypress/e2e/39-projects.cy.ts b/cypress/e2e/39-projects.cy.ts index efd18bca74..b506040c87 100644 --- a/cypress/e2e/39-projects.cy.ts +++ b/cypress/e2e/39-projects.cy.ts @@ -1,3 +1,5 @@ +import { setCredentialValues } from '../composables/modals/credential-modal'; +import { clickCreateNewCredential, selectResourceLocatorItem } from '../composables/ndv'; import * as projects from '../composables/projects'; import { INSTANCE_ADMIN, @@ -11,18 +13,16 @@ import { WorkflowPage, CredentialsModal, CredentialsPage, - WorkflowExecutionsTab, NDV, MainSidebar, } from '../pages'; import { clearNotifications, successToast } from '../pages/notifications'; -import { getVisibleDropdown, getVisibleModalOverlay, getVisibleSelect } from '../utils'; +import { getVisibleSelect } from '../utils'; const workflowsPage = new WorkflowsPage(); const workflowPage = new WorkflowPage(); const credentialsPage = new CredentialsPage(); const credentialsModal = new CredentialsModal(); -const executionsTab = new WorkflowExecutionsTab(); const ndv = new NDV(); const mainSidebar = new MainSidebar(); @@ -36,207 +36,6 @@ describe('Projects', { disableAutoLogin: true }, () => { cy.changeQuota('maxTeamProjects', -1); }); - it('should handle workflows and credentials and menu items', () => { - cy.signinAsAdmin(); - cy.visit(workflowsPage.url); - workflowsPage.getters.workflowCards().should('not.have.length'); - - workflowsPage.getters.newWorkflowButtonCard().click(); - - cy.intercept('POST', '/rest/workflows').as('workflowSave'); - workflowPage.actions.saveWorkflowOnButtonClick(); - - cy.wait('@workflowSave').then((interception) => { - expect(interception.request.body).not.to.have.property('projectId'); - }); - - projects.getHomeButton().click(); - projects.getProjectTabs().should('have.length', 3); - - projects.getProjectTabCredentials().click(); - credentialsPage.getters.credentialCards().should('not.have.length'); - - credentialsPage.getters.emptyListCreateCredentialButton().click(); - credentialsModal.getters.newCredentialModal().should('be.visible'); - credentialsModal.getters.newCredentialTypeSelect().should('be.visible'); - credentialsModal.getters.newCredentialTypeOption('Notion API').click(); - credentialsModal.getters.newCredentialTypeButton().click(); - credentialsModal.getters.connectionParameter('Internal Integration Secret').type('1234567890'); - credentialsModal.actions.setName('My awesome Notion account'); - - cy.intercept('POST', '/rest/credentials').as('credentialSave'); - credentialsModal.actions.save(); - cy.wait('@credentialSave').then((interception) => { - expect(interception.request.body).not.to.have.property('projectId'); - }); - - credentialsModal.actions.close(); - credentialsPage.getters.credentialCards().should('have.length', 1); - credentialsPage.getters - .credentialCards() - .first() - .find('.n8n-node-icon img') - .should('be.visible'); - - projects.getProjectTabWorkflows().click(); - workflowsPage.getters.workflowCards().should('have.length', 1); - - projects.getMenuItems().should('not.have.length'); - - cy.intercept('POST', '/rest/projects').as('projectCreate'); - projects.getAddProjectButton().click(); - cy.wait('@projectCreate'); - projects.getMenuItems().should('have.length', 1); - projects.getProjectTabs().should('have.length', 3); - - cy.get('input[name="name"]').type('Development'); - projects.addProjectMember(INSTANCE_MEMBERS[0].email); - - cy.intercept('PATCH', '/rest/projects/*').as('projectSettingsSave'); - projects.getProjectSettingsSaveButton().click(); - cy.wait('@projectSettingsSave').then((interception) => { - expect(interception.request.body).to.have.property('name').and.to.equal('Development'); - expect(interception.request.body).to.have.property('relations').to.have.lengthOf(2); - }); - - projects.getMenuItems().first().click(); - workflowsPage.getters.workflowCards().should('not.have.length'); - projects.getProjectTabs().should('have.length', 4); - - workflowsPage.getters.newWorkflowButtonCard().click(); - - cy.intercept('POST', '/rest/workflows').as('workflowSave'); - workflowPage.actions.saveWorkflowOnButtonClick(); - - cy.wait('@workflowSave').then((interception) => { - expect(interception.request.body).to.have.property('projectId'); - }); - - projects.getMenuItems().first().click(); - - projects.getProjectTabCredentials().click(); - credentialsPage.getters.credentialCards().should('not.have.length'); - - credentialsPage.getters.emptyListCreateCredentialButton().click(); - credentialsModal.getters.newCredentialModal().should('be.visible'); - credentialsModal.getters.newCredentialTypeSelect().should('be.visible'); - credentialsModal.getters.newCredentialTypeOption('Notion API').click(); - credentialsModal.getters.newCredentialTypeButton().click(); - credentialsModal.getters.connectionParameter('Internal Integration Secret').type('1234567890'); - credentialsModal.actions.setName('My awesome Notion account'); - - cy.intercept('POST', '/rest/credentials').as('credentialSave'); - credentialsModal.actions.save(); - cy.wait('@credentialSave').then((interception) => { - expect(interception.request.body).to.have.property('projectId'); - }); - credentialsModal.actions.close(); - - projects.getAddProjectButton().click(); - projects.getMenuItems().should('have.length', 2); - - let projectId: string; - projects.getMenuItems().first().click(); - cy.intercept('GET', '/rest/credentials*').as('credentialsList'); - projects.getProjectTabCredentials().click(); - cy.wait('@credentialsList').then((interception) => { - const url = new URL(interception.request.url); - const queryParams = new URLSearchParams(url.search); - const filter = queryParams.get('filter'); - expect(filter).to.be.a('string').and.to.contain('projectId'); - - if (filter) { - projectId = JSON.parse(filter).projectId; - } - }); - - projects.getMenuItems().last().click(); - cy.intercept('GET', '/rest/credentials*').as('credentialsListProjectId'); - projects.getProjectTabCredentials().click(); - cy.wait('@credentialsListProjectId').then((interception) => { - const url = new URL(interception.request.url); - const queryParams = new URLSearchParams(url.search); - const filter = queryParams.get('filter'); - expect(filter).to.be.a('string').and.to.contain('projectId'); - - if (filter) { - expect(JSON.parse(filter).projectId).not.to.equal(projectId); - } - }); - - projects.getHomeButton().click(); - workflowsPage.getters.workflowCards().should('have.length', 2); - - cy.intercept('GET', '/rest/credentials*').as('credentialsListUnfiltered'); - projects.getProjectTabCredentials().click(); - cy.wait('@credentialsListUnfiltered').then((interception) => { - expect(interception.request.url).not.to.contain('filter'); - }); - - let menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Overview")[class*=active_]').should('exist'); - - projects.getMenuItems().first().click(); - - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Development")[class*=active_]').should('exist'); - - cy.intercept('GET', '/rest/workflows/*').as('loadWorkflow'); - workflowsPage.getters.workflowCards().first().findChildByTestId('card-content').click(); - - cy.wait('@loadWorkflow'); - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Development")[class*=active_]').should('exist'); - - cy.intercept('GET', '/rest/executions*').as('loadExecutions'); - executionsTab.actions.switchToExecutionsTab(); - - cy.wait('@loadExecutions'); - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Development")[class*=active_]').should('exist'); - - executionsTab.actions.switchToEditorTab(); - - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Development")[class*=active_]').should('exist'); - - cy.getByTestId('menu-item').filter(':contains("Variables")').click(); - cy.getByTestId('unavailable-resources-list').should('be.visible'); - - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Variables")[class*=active_]').should('exist'); - - projects.getHomeButton().click(); - menuItems = cy.getByTestId('menu-item'); - - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Overview")[class*=active_]').should('exist'); - - workflowsPage.getters.workflowCards().should('have.length', 2).first().click(); - - cy.wait('@loadWorkflow'); - cy.getByTestId('execute-workflow-button').should('be.visible'); - - menuItems = cy.getByTestId('menu-item'); - menuItems.filter(':contains("Overview")[class*=active_]').should('not.exist'); - - menuItems = cy.getByTestId('menu-item'); - menuItems.filter('[class*=active_]').should('have.length', 1); - menuItems.filter(':contains("Development")[class*=active_]').should('exist'); - }); - it('should not show project add button and projects to a member if not invited to any project', () => { cy.signinAsMember(1); cy.visit(workflowsPage.url); @@ -245,26 +44,6 @@ describe('Projects', { disableAutoLogin: true }, () => { projects.getMenuItems().should('not.exist'); }); - it('should not show viewer role if not licensed', () => { - cy.signinAsOwner(); - cy.visit(workflowsPage.url); - - projects.getMenuItems().first().click(); - projects.getProjectTabSettings().click(); - - cy.get( - `[data-test-id="user-list-item-${INSTANCE_MEMBERS[0].email}"] [data-test-id="projects-settings-user-role-select"]`, - ).click(); - - cy.get('.el-select-dropdown__item.is-disabled') - .should('contain.text', 'Viewer') - .get('span:contains("Upgrade")') - .filter(':visible') - .click(); - - getVisibleModalOverlay().should('contain.text', 'Upgrade to unlock additional roles'); - }); - describe('when starting from scratch', () => { beforeEach(() => { cy.resetDatabase(); @@ -275,7 +54,11 @@ describe('Projects', { disableAutoLogin: true }, () => { cy.changeQuota('maxTeamProjects', -1); }); - it('should filter credentials by project ID when creating new workflow or hard reloading an opened workflow', () => { + /** + * @TODO: New Canvas - Fix this test + */ + // eslint-disable-next-line n8n-local-rules/no-skipped-tests + it.skip('should filter credentials by project ID when creating new workflow or hard reloading an opened workflow', () => { cy.signinAsOwner(); cy.visit(workflowsPage.url); @@ -367,7 +150,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account project 1'); ndv.getters.backToCanvas().click(); @@ -382,7 +165,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account project 1'); ndv.getters.backToCanvas().click(); @@ -396,7 +179,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account project 2'); ndv.getters.backToCanvas().click(); @@ -407,7 +190,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account project 2'); ndv.getters.backToCanvas().click(); @@ -425,7 +208,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account personal project'); ndv.getters.backToCanvas().click(); @@ -436,7 +219,7 @@ describe('Projects', { disableAutoLogin: true }, () => { workflowPage.getters.nodeCredentialsSelect().first().click(); getVisibleSelect() .find('li') - .should('have.length', 2) + .should('have.length', 1) .first() .should('contain.text', 'Notion account personal project'); }); @@ -754,80 +537,64 @@ describe('Projects', { disableAutoLogin: true }, () => { ndv.getters.credentialInput().find('input').should('be.enabled'); }); - it('should handle viewer role', () => { - cy.enableFeature('projectRole:viewer'); + it('should create sub-workflow and credential in the sub-workflow in the same project', () => { cy.signinAsOwner(); cy.visit(workflowsPage.url); - projects.createProject('Development'); - projects.addProjectMember(INSTANCE_MEMBERS[0].email, 'Viewer'); - projects.getProjectSettingsSaveButton().click(); - + projects.createProject('Dev'); projects.getProjectTabWorkflows().click(); workflowsPage.getters.newWorkflowButtonCard().click(); - projects.createWorkflow('Test_workflow_4_executions_view.json', 'WF with random error'); - executionsTab.actions.createManualExecutions(2); - executionsTab.actions.toggleNodeEnabled('Error'); - executionsTab.actions.createManualExecutions(2); - workflowPage.actions.saveWorkflowUsingKeyboardShortcut(); + workflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); + workflowPage.actions.saveWorkflowOnButtonClick(); + workflowPage.actions.addNodeToCanvas('Execute Workflow', true, true); - projects.getMenuItems().first().click(); - projects.getProjectTabCredentials().click(); - credentialsPage.getters.emptyListCreateCredentialButton().click(); - projects.createCredential('Notion API'); + cy.window().then((win) => { + cy.stub(win, 'open').callsFake((url) => { + cy.visit(url); + }); + }); - mainSidebar.actions.openUserMenu(); - cy.getByTestId('user-menu-item-logout').click(); + selectResourceLocatorItem('workflowId', 0, 'Create a'); - cy.get('input[name="email"]').type(INSTANCE_MEMBERS[0].email); - cy.get('input[name="password"]').type(INSTANCE_MEMBERS[0].password); - cy.getByTestId('form-submit-button').click(); + cy.get('body').type('{esc}'); + workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); + clickCreateNewCredential(); + setCredentialValues({ + apiKey: 'abc123', + }); + ndv.actions.close(); + workflowPage.actions.saveWorkflowOnButtonClick(); projects.getMenuItems().last().click(); - projects.getProjectTabExecutions().click(); - cy.getByTestId('global-execution-list-item').first().find('td:last button').click(); - getVisibleDropdown() - .find('li') - .filter(':contains("Retry")') - .should('have.class', 'is-disabled'); - getVisibleDropdown() - .find('li') - .filter(':contains("Delete")') - .should('have.class', 'is-disabled'); + workflowsPage.getters.workflowCards().should('have.length', 2); - projects.getMenuItems().first().click(); - cy.getByTestId('workflow-card-name').should('be.visible').first().click(); - workflowPage.getters.nodeViewRoot().should('be.visible'); - workflowPage.getters.executeWorkflowButton().should('not.exist'); - workflowPage.getters.nodeCreatorPlusButton().should('not.exist'); - workflowPage.getters.canvasNodes().should('have.length', 3).last().click(); - cy.get('body').type('{backspace}'); - workflowPage.getters.canvasNodes().should('have.length', 3).last().rightclick(); - getVisibleDropdown() - .find('li') - .should('be.visible') - .filter( - ':contains("Open"), :contains("Copy"), :contains("Select all"), :contains("Clear selection")', - ) - .should('not.have.class', 'is-disabled'); - cy.get('body').type('{esc}'); - - executionsTab.actions.switchToExecutionsTab(); - cy.getByTestId('retry-execution-button') - .should('be.visible') - .find('.is-disabled') - .should('exist'); - cy.get('button:contains("Debug")').should('be.disabled'); - cy.get('button[title="Retry execution"]').should('be.disabled'); - cy.get('button[title="Delete this execution"]').should('be.disabled'); - - projects.getMenuItems().first().click(); projects.getProjectTabCredentials().click(); - credentialsPage.getters.credentialCards().filter(':contains("Notion")').click(); - cy.getByTestId('node-credentials-config-container') - .should('be.visible') - .find('input') - .should('not.have.length'); + credentialsPage.getters.credentialCards().should('have.length', 1); + }); + + it('should create credential from workflow in the correct project after editor page refresh', () => { + cy.signinAsOwner(); + cy.visit(workflowsPage.url); + + projects.createProject('Dev'); + projects.getProjectTabWorkflows().click(); + workflowsPage.getters.newWorkflowButtonCard().click(); + workflowPage.actions.addNodeToCanvas(MANUAL_TRIGGER_NODE_NAME); + workflowPage.actions.saveWorkflowOnButtonClick(); + + cy.reload(); + + workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true); + clickCreateNewCredential(); + setCredentialValues({ + apiKey: 'abc123', + }); + ndv.actions.close(); + workflowPage.actions.saveWorkflowOnButtonClick(); + + projects.getMenuItems().last().click(); + projects.getProjectTabCredentials().click(); + credentialsPage.getters.credentialCards().should('have.length', 1); }); }); diff --git a/cypress/e2e/4-node-creator.cy.ts b/cypress/e2e/4-node-creator.cy.ts index 5e32d5568c..5636179166 100644 --- a/cypress/e2e/4-node-creator.cy.ts +++ b/cypress/e2e/4-node-creator.cy.ts @@ -76,11 +76,21 @@ describe('Node Creator', () => { nodeCreatorFeature.getters.canvasAddButton().click(); WorkflowPage.actions.addNodeToCanvas('Manual', false); - nodeCreatorFeature.getters.canvasAddButton().should('not.be.visible'); - nodeCreatorFeature.getters.nodeCreator().should('not.exist'); + cy.ifCanvasVersion( + () => { + nodeCreatorFeature.getters.canvasAddButton().should('not.be.visible'); + nodeCreatorFeature.getters.nodeCreator().should('not.exist'); + // TODO: Replace once we have canvas feature utils + cy.get('div').contains('Add first step').should('be.hidden'); + }, + () => { + nodeCreatorFeature.getters.canvasAddButton().should('not.exist'); + nodeCreatorFeature.getters.nodeCreator().should('not.exist'); + // TODO: Replace once we have canvas feature utils + cy.get('div').contains('Add first step').should('not.exist'); + }, + ); - // TODO: Replace once we have canvas feature utils - cy.get('div').contains('Add first step').should('be.hidden'); nodeCreatorFeature.actions.openNodeCreator(); nodeCreatorFeature.getters.nodeCreator().contains('What happens next?').should('be.visible'); @@ -125,7 +135,6 @@ describe('Node Creator', () => { 'OpenThesaurus', 'Spontit', 'Vonage', - 'Send Email', 'Toggl Trigger', ]; const doubleActionNode = 'OpenWeatherMap'; @@ -346,7 +355,15 @@ describe('Node Creator', () => { it('should correctly append a No Op node when Loop Over Items node is added (from connection)', () => { WorkflowPage.actions.addNodeToCanvas('Manual'); - cy.get('.plus-endpoint').should('be.visible').click(); + + cy.ifCanvasVersion( + () => { + cy.get('.plus-endpoint').click(); + }, + () => { + cy.getByTestId('canvas-handle-plus').click(); + }, + ); nodeCreatorFeature.getters.searchBar().find('input').type('Loop Over Items'); nodeCreatorFeature.getters.getCreatorItem('Loop Over Items').click(); @@ -531,7 +548,7 @@ describe('Node Creator', () => { vectorStores.each((_i, vectorStore) => { nodeCreatorFeature.getters.getCreatorItem(vectorStore).click(); actions.forEach((action) => { - nodeCreatorFeature.getters.getCreatorItem(action).should('be.visible'); + nodeCreatorFeature.getters.getCreatorItem(action).should('be.visible').realHover(); }); cy.realPress('ArrowLeft'); }); @@ -554,4 +571,13 @@ describe('Node Creator', () => { addVectorStoreToolToParent('In-Memory Vector Store', AGENT_NODE_NAME); }); + + it('should insert node to canvas with sendAndWait operation selected', () => { + nodeCreatorFeature.getters.canvasAddButton().click(); + WorkflowPage.actions.addNodeToCanvas('Manual', false); + nodeCreatorFeature.actions.openNodeCreator(); + cy.contains('Human in the loop').click(); + nodeCreatorFeature.getters.getCreatorItem('Slack').click(); + cy.contains('Send and Wait for Response').should('exist'); + }); }); diff --git a/cypress/e2e/45-ai-assistant.cy.ts b/cypress/e2e/45-ai-assistant.cy.ts index 9d33811363..157c656b46 100644 --- a/cypress/e2e/45-ai-assistant.cy.ts +++ b/cypress/e2e/45-ai-assistant.cy.ts @@ -5,7 +5,6 @@ import { GMAIL_NODE_NAME, SCHEDULE_TRIGGER_NODE_NAME } from '../constants'; import { CredentialsModal, CredentialsPage, NDV, WorkflowPage } from '../pages'; import { AIAssistant } from '../pages/features/ai-assistant'; import { NodeCreator } from '../pages/features/node-creator'; -import { getVisibleSelect } from '../utils'; const wf = new WorkflowPage(); const ndv = new NDV(); @@ -434,7 +433,7 @@ describe('AI Assistant Credential Help', () => { wf.actions.addNodeToCanvas('Slack', true, true, 'Get a channel'); wf.getters.nodeCredentialsSelect().should('exist'); wf.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + wf.getters.nodeCredentialsCreateOption().click(); credentialsModal.getters.credentialAuthTypeRadioButtons().first().click(); ndv.getters.copyInput().should('not.exist'); credentialsModal.getters.oauthConnectButton().should('have.length', 1); @@ -467,7 +466,7 @@ describe('AI Assistant Credential Help', () => { wf.actions.addNodeToCanvas('Microsoft Outlook', true, true, 'Get a calendar'); wf.getters.nodeCredentialsSelect().should('exist'); wf.getters.nodeCredentialsSelect().click(); - getVisibleSelect().find('li').last().click(); + wf.getters.nodeCredentialsCreateOption().click(); ndv.getters.copyInput().should('not.exist'); credentialsModal.getters.oauthConnectButton().should('have.length', 1); credentialsModal.getters.credentialInputs().should('have.length', 1); diff --git a/cypress/e2e/45-workflow-selector-parameter.cy.ts b/cypress/e2e/45-workflow-selector-parameter.cy.ts index 38df9a29b8..40e5ca2488 100644 --- a/cypress/e2e/45-workflow-selector-parameter.cy.ts +++ b/cypress/e2e/45-workflow-selector-parameter.cy.ts @@ -17,7 +17,7 @@ describe('Workflow Selector Parameter', () => { workflowPage.actions.visit(); workflowPage.actions.addInitialNodeToCanvas(EXECUTE_WORKFLOW_NODE_NAME, { keepNdvOpen: true, - action: 'Call Another Workflow', + action: 'Execute A Sub Workflow', }); }); it('should render sub-workflows list', () => { @@ -86,6 +86,8 @@ describe('Workflow Selector Parameter', () => { cy.stub(win, 'open').as('windowOpen'); }); + cy.intercept('POST', '/rest/workflows*').as('createSubworkflow'); + ndv.getters.resourceLocator('workflowId').should('be.visible'); ndv.getters.resourceLocatorInput('workflowId').click(); @@ -94,14 +96,24 @@ describe('Workflow Selector Parameter', () => { .findChildByTestId('rlc-item') .eq(0) .find('span') - .should('have.text', 'Create a new sub-workflow'); + .should('contain.text', 'Create a'); // Due to some inconsistency we're sometimes in a project and sometimes not, this covers both cases getVisiblePopper().findChildByTestId('rlc-item').eq(0).click(); - const SAMPLE_SUBWORKFLOW_TEMPLATE_ID = 'VMiAxXa3lCAizGB5f7dVZQSFfg3FtHkdTKvLuupqBls='; - cy.get('@windowOpen').should( - 'be.calledWith', - `/workflows/onboarding/${SAMPLE_SUBWORKFLOW_TEMPLATE_ID}?sampleSubWorkflows=0`, - ); + cy.wait('@createSubworkflow').then((interception) => { + expect(interception.request.body).to.have.property('name').that.includes('Sub-Workflow'); + expect(interception.request.body.nodes).to.be.an('array'); + expect(interception.request.body.nodes).to.have.length(2); + expect(interception.request.body.nodes[0]).to.have.property( + 'name', + 'When Executed by Another Workflow', + ); + expect(interception.request.body.nodes[1]).to.have.property( + 'name', + 'Replace me with your logic', + ); + }); + + cy.get('@windowOpen').should('be.calledWithMatch', /\/workflow\/.+/); }); }); diff --git a/cypress/e2e/47-subworkflow-debugging.cy.ts b/cypress/e2e/47-subworkflow-debugging.cy.ts index 725b6b32c4..77aaa4d7f6 100644 --- a/cypress/e2e/47-subworkflow-debugging.cy.ts +++ b/cypress/e2e/47-subworkflow-debugging.cy.ts @@ -1,9 +1,3 @@ -import { - getExecutionPreviewOutputPanelRelatedExecutionLink, - getExecutionsSidebar, - getWorkflowExecutionPreviewIframe, - openExecutionPreviewNode, -} from '../composables/executions'; import { changeOutputRunSelector, getOutputPanelItemsCount, @@ -103,38 +97,4 @@ describe('Subworkflow debugging', () => { getOutputTbodyCell(1, 2).should('include.text', 'Terry.Dach@hotmail.com'); }); }); - - it('can inspect parent executions', () => { - cy.url().then((workflowUrl) => { - openNode('Execute Workflow with param'); - - getOutputPanelItemsCount().should('contain.text', '2 items, 1 sub-execution'); - getOutputPanelRelatedExecutionLink().should('contain.text', 'View sub-execution'); - getOutputPanelRelatedExecutionLink().should('have.attr', 'href'); - - // ensure workflow executed and waited on output - getOutputTableHeaders().should('have.length', 2); - getOutputTbodyCell(1, 0).should('have.text', 'world Natalie Moore'); - - // cypress cannot handle new tabs so removing it - getOutputPanelRelatedExecutionLink().invoke('removeAttr', 'target').click(); - - getExecutionsSidebar().should('be.visible'); - getWorkflowExecutionPreviewIframe().should('be.visible'); - openExecutionPreviewNode('Execute Workflow Trigger'); - - getExecutionPreviewOutputPanelRelatedExecutionLink().should( - 'include.text', - 'View parent execution', - ); - - getExecutionPreviewOutputPanelRelatedExecutionLink() - .invoke('removeAttr', 'target') - .click({ force: true }); - - cy.url().then((currentUrl) => { - expect(currentUrl === workflowUrl); - }); - }); - }); }); diff --git a/cypress/e2e/48-subworkflow-inputs.cy.ts b/cypress/e2e/48-subworkflow-inputs.cy.ts index 0e2755b9f0..716d253b8c 100644 --- a/cypress/e2e/48-subworkflow-inputs.cy.ts +++ b/cypress/e2e/48-subworkflow-inputs.cy.ts @@ -1,60 +1,227 @@ -import { clickGetBackToCanvas, getOutputTableHeaders } from '../composables/ndv'; import { + addItemToFixedCollection, + assertNodeOutputHintExists, + clickExecuteNode, + clickGetBackToCanvas, + getExecuteNodeButton, + getOutputTableHeaders, + getParameterInputByName, + populateFixedCollection, + selectResourceLocatorItem, + typeIntoFixedCollectionItem, + clickWorkflowCardContent, + assertOutputTableContent, + populateMapperFields, + getNodeRunInfoStale, + assertNodeOutputErrorMessageExists, + checkParameterCheckboxInputByName, + uncheckParameterCheckboxInputByName, +} from '../composables/ndv'; +import { + clickExecuteWorkflowButton, clickZoomToFit, navigateToNewWorkflowPage, openNode, pasteWorkflow, saveWorkflowOnButtonClick, } from '../composables/workflow'; +import { visitWorkflowsPage } from '../composables/workflowsPage'; import SUB_WORKFLOW_INPUTS from '../fixtures/Test_Subworkflow-Inputs.json'; -import { NDV, WorkflowsPage, WorkflowPage } from '../pages'; import { errorToast, successToast } from '../pages/notifications'; import { getVisiblePopper } from '../utils'; -const ndv = new NDV(); -const workflowsPage = new WorkflowsPage(); -const workflow = new WorkflowPage(); - const DEFAULT_WORKFLOW_NAME = 'My workflow'; const DEFAULT_SUBWORKFLOW_NAME_1 = 'My Sub-Workflow 1'; const DEFAULT_SUBWORKFLOW_NAME_2 = 'My Sub-Workflow 2'; -type FieldRow = readonly string[]; - -const exampleFields = [ +const EXAMPLE_FIELDS = [ ['aNumber', 'Number'], ['aString', 'String'], ['aArray', 'Array'], ['aObject', 'Object'], ['aAny', 'Allow Any Type'], - // bool last since it's not an inputField so we'll skip it for some cases + // bool last because it's a switch instead of a normal inputField so we'll skip it for some cases ['aBool', 'Boolean'], ] as const; -/** - * Populate multiValue fixedCollections. Only supports fixedCollections for which all fields can be defined via keyboard typing - * - * @param items - 2D array of items to populate, i.e. [["myField1", "String"], [""] - * @param collectionName - name of the fixedCollection to populate - * @param offset - amount of 'parameter-input's before the fixedCollection under test - * @returns - */ -function populateFixedCollection( - items: readonly FieldRow[], - collectionName: string, - offset: number, -) { - if (items.length === 0) return; - const n = items[0].length; - for (const [i, params] of items.entries()) { - ndv.actions.addItemToFixedCollection(collectionName); - for (const [j, param] of params.entries()) { - ndv.getters - .fixedCollectionParameter(collectionName) - .getByTestId('parameter-input') - .eq(offset + i * n + j) - .type(`${param}{downArrow}{enter}`); - } +type TypeField = 'Allow Any Type' | 'String' | 'Number' | 'Boolean' | 'Array' | 'Object'; + +describe('Sub-workflow creation and typed usage', () => { + beforeEach(() => { + navigateToNewWorkflowPage(); + pasteWorkflow(SUB_WORKFLOW_INPUTS); + saveWorkflowOnButtonClick(); + clickZoomToFit(); + + openNode('Execute Workflow'); + + // Prevent sub-workflow from opening in new window + cy.window().then((win) => { + cy.stub(win, 'open').callsFake((url) => { + cy.visit(url); + }); + }); + selectResourceLocatorItem('workflowId', 0, 'Create a'); + // ************************** + // NAVIGATE TO CHILD WORKFLOW + // ************************** + // Close NDV before opening the node creator + cy.get('body').type('{esc}'); + openNode('When Executed by Another Workflow'); + }); + + it('works with type-checked values', () => { + populateFixedCollection(EXAMPLE_FIELDS, 'workflowInputs', 1); + + validateAndReturnToParent( + DEFAULT_SUBWORKFLOW_NAME_1, + 1, + EXAMPLE_FIELDS.map((f) => f[0]), + ); + + const values = [ + '-1', // number fields don't support `=` switch to expression, so let's test the Fixed case with it + ...EXAMPLE_FIELDS.slice(1).map((x) => `={{}{{} $json.a${x[0]}`), // the `}}` at the end are added automatically + ]; + + // this matches with the pinned data provided in the fixture + populateMapperFields(values.map((x, i) => [EXAMPLE_FIELDS[i][0], x])); + + clickExecuteNode(); + + const expected = [ + ['-1', 'A String', '0:11:true2:3', 'aKey:-1', '[empty object]', 'true'], + ['-1', 'Another String', '[empty array]', 'aDifferentKey:-1', '[empty array]', 'true'], + ]; + assertOutputTableContent(expected); + + // Test the type-checking options + populateMapperFields([['aString', '{selectAll}{backspace}{{}{{} 5']]); + + getNodeRunInfoStale().should('exist'); + clickExecuteNode(); + + assertNodeOutputErrorMessageExists(); + + // attemptToConvertTypes enabled + checkParameterCheckboxInputByName('attemptToConvertTypes'); + + getNodeRunInfoStale().should('exist'); + clickExecuteNode(); + + const expected2 = [ + ['-1', '5', '0:11:true2:3', 'aKey:-1', '[empty object]', 'true'], + ['-1', '5', '[empty array]', 'aDifferentKey:-1', '[empty array]', 'true'], + ]; + + assertOutputTableContent(expected2); + + // disabled again + uncheckParameterCheckboxInputByName('attemptToConvertTypes'); + + getNodeRunInfoStale().should('exist'); + clickExecuteNode(); + + assertNodeOutputErrorMessageExists(); + }); + + it('works with Fields input source, and can then be changed to JSON input source', () => { + assertNodeOutputHintExists(); + + populateFixedCollection(EXAMPLE_FIELDS, 'workflowInputs', 1); + + validateAndReturnToParent( + DEFAULT_SUBWORKFLOW_NAME_1, + 1, + EXAMPLE_FIELDS.map((f) => f[0]), + ); + + cy.window().then((win) => { + cy.stub(win, 'open').callsFake((url) => { + cy.visit(url); + selectResourceLocatorItem('workflowId', 0, 'Create a'); + + openNode('When Executed by Another Workflow'); + + getParameterInputByName('inputSource').click(); + + getVisiblePopper() + .getByTestId('parameter-input') + .eq(0) + .type('Using JSON Example{downArrow}{enter}'); + + const exampleJson = + '{{}' + EXAMPLE_FIELDS.map((x) => `"${x[0]}": ${makeExample(x[1])}`).join(',') + '}'; + getParameterInputByName('jsonExample') + .find('.cm-line') + .eq(0) + .type(`{selectAll}{backspace}${exampleJson}{enter}`); + + // first one doesn't work for some reason, might need to wait for something? + clickExecuteNode(); + + validateAndReturnToParent( + DEFAULT_SUBWORKFLOW_NAME_2, + 2, + EXAMPLE_FIELDS.map((f) => f[0]), + ); + + assertOutputTableContent([ + ['[null]', '[null]', '[null]', '[null]', '[null]', 'true'], + ['[null]', '[null]', '[null]', '[null]', '[null]', 'true'], + ]); + + clickExecuteNode(); + }); + }); + }); + + it('should show node issue when no fields are defined in manual mode', () => { + getExecuteNodeButton().should('be.disabled'); + clickGetBackToCanvas(); + // Executing the workflow should show an error toast + clickExecuteWorkflowButton(); + errorToast().should('contain', 'The workflow has issues'); + openNode('When Executed by Another Workflow'); + // Add a field to the workflowInputs fixedCollection + addItemToFixedCollection('workflowInputs'); + typeIntoFixedCollectionItem('workflowInputs', 0, 'test'); + // Executing the workflow should not show error now + clickGetBackToCanvas(); + clickExecuteWorkflowButton(); + successToast().should('contain', 'Workflow executed successfully'); + }); +}); + +// This function starts off in the Child Workflow Input Trigger, assuming we just defined the input fields +// It then navigates back to the parent and validates the outputPanel matches our changes +function validateAndReturnToParent(targetChild: string, offset: number, fields: string[]) { + clickExecuteNode(); + + // + 1 to account for formatting-only column + getOutputTableHeaders().should('have.length', fields.length + 1); + for (const [i, name] of fields.entries()) { + getOutputTableHeaders().eq(i).should('have.text', name); + } + + clickGetBackToCanvas(); + saveWorkflowOnButtonClick(); + + visitWorkflowsPage(); + + clickWorkflowCardContent(DEFAULT_WORKFLOW_NAME); + + openNode('Execute Workflow'); + + // Note that outside of e2e tests this will be pre-selected correctly. + // Due to our workaround to remain in the same tab we need to select the correct tab manually + selectResourceLocatorItem('workflowId', offset, targetChild); + + clickExecuteNode(); + + getOutputTableHeaders().should('have.length', fields.length + 1); + for (const [i, name] of fields.entries()) { + getOutputTableHeaders().eq(i).should('have.text', name); } } @@ -74,215 +241,3 @@ function makeExample(type: TypeField) { return 'null'; } } - -type TypeField = 'Allow Any Type' | 'String' | 'Number' | 'Boolean' | 'Array' | 'Object'; -function populateFields(items: ReadonlyArray) { - populateFixedCollection(items, 'workflowInputs', 1); -} - -function navigateWorkflowSelectionDropdown(index: number, expectedText: string) { - ndv.getters.resourceLocator('workflowId').should('be.visible'); - ndv.getters.resourceLocatorInput('workflowId').click(); - - getVisiblePopper().findChildByTestId('rlc-item').eq(0).should('exist'); - getVisiblePopper() - .findChildByTestId('rlc-item') - .eq(index) - .find('span') - .should('have.text', expectedText) - .click(); -} - -function populateMapperFields(values: readonly string[], offset: number) { - for (const [i, value] of values.entries()) { - cy.getByTestId('parameter-input') - .eq(offset + i) - .type(value); - - // Click on a parent to dismiss the pop up hiding the field below. - cy.getByTestId('parameter-input') - .eq(offset + i) - .parent() - .parent() - .click('topLeft'); - } -} - -// This function starts off in the Child Workflow Input Trigger, assuming we just defined the input fields -// It then navigates back to the parent and validates output -function validateAndReturnToParent(targetChild: string, offset: number, fields: string[]) { - ndv.actions.execute(); - - // + 1 to account for formatting-only column - getOutputTableHeaders().should('have.length', fields.length + 1); - for (const [i, name] of fields.entries()) { - getOutputTableHeaders().eq(i).should('have.text', name); - } - - clickGetBackToCanvas(); - saveWorkflowOnButtonClick(); - - cy.visit(workflowsPage.url); - - workflowsPage.getters.workflowCardContent(DEFAULT_WORKFLOW_NAME).click(); - - openNode('Execute Workflow'); - - // Note that outside of e2e tests this will be pre-selected correctly. - // Due to our workaround to remain in the same tab we need to select the correct tab manually - navigateWorkflowSelectionDropdown(offset, targetChild); - - // This fails, pointing to `usePushConnection` `const triggerNode = subWorkflow?.nodes.find` being `undefined.find()`I - ndv.actions.execute(); - - getOutputTableHeaders().should('have.length', fields.length + 1); - for (const [i, name] of fields.entries()) { - getOutputTableHeaders().eq(i).should('have.text', name); - } - - // todo: verify the fields appear and show the correct types - - // todo: fill in the input fields (and mock previous node data in the json fixture to match) - - // todo: validate the actual output data -} - -function setWorkflowInputFieldValue(index: number, value: string) { - ndv.actions.addItemToFixedCollection('workflowInputs'); - ndv.actions.typeIntoFixedCollectionItem('workflowInputs', index, value); -} - -describe('Sub-workflow creation and typed usage', () => { - beforeEach(() => { - navigateToNewWorkflowPage(); - pasteWorkflow(SUB_WORKFLOW_INPUTS); - saveWorkflowOnButtonClick(); - clickZoomToFit(); - - openNode('Execute Workflow'); - - // Prevent sub-workflow from opening in new window - cy.window().then((win) => { - cy.stub(win, 'open').callsFake((url) => { - cy.visit(url); - }); - }); - navigateWorkflowSelectionDropdown(0, 'Create a new sub-workflow'); - // ************************** - // NAVIGATE TO CHILD WORKFLOW - // ************************** - - openNode('Workflow Input Trigger'); - }); - - it('works with type-checked values', () => { - populateFields(exampleFields); - - validateAndReturnToParent( - DEFAULT_SUBWORKFLOW_NAME_1, - 1, - exampleFields.map((f) => f[0]), - ); - - const values = [ - '-1', // number fields don't support `=` switch to expression, so let's test the Fixed case with it - ...exampleFields.slice(1).map((x) => `={{}{{} $json.a${x[0]}`), // }} are added automatically - ]; - - // this matches with the pinned data provided in the fixture - populateMapperFields(values, 2); - - ndv.actions.execute(); - - // todo: - // - validate output lines up - // - change input to need casts - // - run - // - confirm error - // - switch `attemptToConvertTypes` flag - // - confirm success and changed output - // - change input to be invalid despite cast - // - run - // - confirm error - // - switch type option flags - // - run - // - confirm success - // - turn off attempt to cast flag - // - confirm a value was not cast - }); - - it('works with Fields input source into JSON input source', () => { - ndv.getters.nodeOutputHint().should('exist'); - - populateFields(exampleFields); - - validateAndReturnToParent( - DEFAULT_SUBWORKFLOW_NAME_1, - 1, - exampleFields.map((f) => f[0]), - ); - - cy.window().then((win) => { - cy.stub(win, 'open').callsFake((url) => { - cy.visit(url); - }); - }); - navigateWorkflowSelectionDropdown(0, 'Create a new sub-workflow'); - - openNode('Workflow Input Trigger'); - - cy.getByTestId('parameter-input').eq(0).click(); - - // Todo: Check if there's a better way to interact with option dropdowns - // This PR would add this child testId - getVisiblePopper() - .getByTestId('parameter-input') - .eq(0) - .type('Using JSON Example{downArrow}{enter}'); - - const exampleJson = - '{{}' + exampleFields.map((x) => `"${x[0]}": ${makeExample(x[1])}`).join(',') + '}'; - cy.getByTestId('parameter-input-jsonExample') - .find('.cm-line') - .eq(0) - .type(`{selectAll}{backspace}${exampleJson}{enter}`); - - // first one doesn't work for some reason, might need to wait for something? - ndv.actions.execute(); - - validateAndReturnToParent( - DEFAULT_SUBWORKFLOW_NAME_2, - 2, - exampleFields.map((f) => f[0]), - ); - - // test for either InputSource mode and options combinations: - // + we're showing the notice in the output panel - // + we start with no fields - // + Test Step works and we create the fields - // + create field of each type (string, number, boolean, object, array, any) - // + exit ndv - // + save - // + go back to parent workflow - // - verify fields appear [needs Ivan's PR] - // - link fields [needs Ivan's PR] - // + run parent - // - verify output with `null` defaults exists - // - }); - - it('should show node issue when no fields are defined in manual mode', () => { - ndv.getters.nodeExecuteButton().should('be.disabled'); - ndv.actions.close(); - // Executing the workflow should show an error toast - workflow.actions.executeWorkflow(); - errorToast().should('contain', 'The workflow has issues'); - openNode('Workflow Input Trigger'); - // Add a field to the workflowInputs fixedCollection - setWorkflowInputFieldValue(0, 'test'); - // Executing the workflow should not show error now - ndv.actions.close(); - workflow.actions.executeWorkflow(); - successToast().should('contain', 'Workflow executed successfully'); - }); -}); diff --git a/cypress/e2e/5-ndv.cy.ts b/cypress/e2e/5-ndv.cy.ts index 8bad424554..22d5b9f49e 100644 --- a/cypress/e2e/5-ndv.cy.ts +++ b/cypress/e2e/5-ndv.cy.ts @@ -1,6 +1,10 @@ import { setCredentialValues } from '../composables/modals/credential-modal'; -import { clickCreateNewCredential } from '../composables/ndv'; -import { MANUAL_TRIGGER_NODE_DISPLAY_NAME, NOTION_NODE_NAME } from '../constants'; +import { clickCreateNewCredential, setParameterSelectByContent } from '../composables/ndv'; +import { + EDIT_FIELDS_SET_NODE_NAME, + MANUAL_TRIGGER_NODE_DISPLAY_NAME, + NOTION_NODE_NAME, +} from '../constants'; import { NDV, WorkflowPage } from '../pages'; import { NodeCreator } from '../pages/features/node-creator'; @@ -106,7 +110,10 @@ describe('NDV', () => { ndv.actions.execute(); ndv.getters .nodeRunErrorMessage() - .should('have.text', 'Info for expression missing from previous node'); + .should( + 'have.text', + "Using the item method doesn't work with pinned data in this scenario. Please unpin 'Break pairedItem chain' and try again.", + ); ndv.getters .nodeRunErrorDescription() .should( @@ -242,6 +249,15 @@ describe('NDV', () => { ndv.actions.switchInputMode('Table'); ndv.actions.switchOutputMode('Table'); + // Start from linked state + ndv.getters.outputLinkRun().then(($el) => { + const classList = Array.from($el[0].classList); + if (!classList.includes('linked')) { + ndv.actions.toggleOutputRunLinking(); + ndv.getters.inputTbodyCell(1, 0).click(); // remove tooltip + } + }); + ndv.getters .inputRunSelector() .should('exist') @@ -359,15 +375,71 @@ describe('NDV', () => { ndv.getters.nodeExecuteButton().should('be.visible'); }); - it('should allow editing code in fullscreen in the Code node', () => { + it('should allow editing code in fullscreen in the code editors', () => { + // Code (JavaScript) workflowPage.actions.addInitialNodeToCanvas('Code', { keepNdvOpen: true }); ndv.actions.openCodeEditorFullscreen(); ndv.getters.codeEditorFullscreen().type('{selectall}').type('{backspace}').type('foo()'); ndv.getters.codeEditorFullscreen().should('contain.text', 'foo()'); - cy.wait(200); + cy.wait(200); // allow change to emit before closing modal ndv.getters.codeEditorDialog().find('.el-dialog__close').click(); ndv.getters.parameterInput('jsCode').get('.cm-content').should('contain.text', 'foo()'); + ndv.actions.close(); + + // SQL + workflowPage.actions.addNodeToCanvas('Postgres', true, true, 'Execute a SQL query'); + ndv.actions.openCodeEditorFullscreen(); + + ndv.getters + .codeEditorFullscreen() + .type('{selectall}') + .type('{backspace}') + .type('SELECT * FROM workflows'); + ndv.getters.codeEditorFullscreen().should('contain.text', 'SELECT * FROM workflows'); + cy.wait(200); + ndv.getters.codeEditorDialog().find('.el-dialog__close').click(); + ndv.getters + .parameterInput('query') + .get('.cm-content') + .should('contain.text', 'SELECT * FROM workflows'); + ndv.actions.close(); + + // HTML + workflowPage.actions.addNodeToCanvas('HTML', true, true, 'Generate HTML template'); + ndv.actions.openCodeEditorFullscreen(); + + ndv.getters + .codeEditorFullscreen() + .type('{selectall}') + .type('{backspace}') + .type('
Hello World'); + ndv.getters.codeEditorFullscreen().should('contain.text', '
Hello World
'); + cy.wait(200); + + ndv.getters.codeEditorDialog().find('.el-dialog__close').click(); + ndv.getters + .parameterInput('html') + .get('.cm-content') + .should('contain.text', '
Hello World
'); + ndv.actions.close(); + + // JSON + workflowPage.actions.addNodeToCanvas(EDIT_FIELDS_SET_NODE_NAME, true, true); + setParameterSelectByContent('mode', 'JSON'); + ndv.actions.openCodeEditorFullscreen(); + ndv.getters + .codeEditorFullscreen() + .type('{selectall}') + .type('{backspace}') + .type('{ "key": "value" }', { parseSpecialCharSequences: false }); + ndv.getters.codeEditorFullscreen().should('contain.text', '{ "key": "value" }'); + cy.wait(200); + ndv.getters.codeEditorDialog().find('.el-dialog__close').click(); + ndv.getters + .parameterInput('jsonOutput') + .get('.cm-content') + .should('contain.text', '{ "key": "value" }'); }); it('should not retrieve remote options when a parameter value changes', () => { diff --git a/cypress/e2e/6-code-node.cy.ts b/cypress/e2e/6-code-node.cy.ts index 674d91af18..4ecda7d56d 100644 --- a/cypress/e2e/6-code-node.cy.ts +++ b/cypress/e2e/6-code-node.cy.ts @@ -7,6 +7,9 @@ import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; const WorkflowPage = new WorkflowPageClass(); const ndv = new NDV(); +const getParameter = () => ndv.getters.parameterInput('jsCode').should('be.visible'); +const getEditor = () => getParameter().find('.cm-content').should('exist'); + describe('Code node', () => { describe('Code editor', () => { beforeEach(() => { @@ -40,10 +43,23 @@ describe('Code node', () => { successToast().contains('Node executed successfully'); }); - it('should show lint errors in `runOnceForAllItems` mode', () => { - const getParameter = () => ndv.getters.parameterInput('jsCode').should('be.visible'); - const getEditor = () => getParameter().find('.cm-content').should('exist'); + it('should allow switching between sibling code nodes', () => { + // Setup + getEditor().type('{selectall}').paste("console.log('code node 1')"); + ndv.actions.close(); + WorkflowPage.actions.addNodeToCanvas('Code', true, true); + getEditor().type('{selectall}').paste("console.log('code node 2')"); + ndv.actions.close(); + WorkflowPage.actions.openNode('Code'); + ndv.actions.clickFloatingNode('Code1'); + getEditor().should('have.text', "console.log('code node 2')"); + + ndv.actions.clickFloatingNode('Code'); + getEditor().should('have.text', "console.log('code node 1')"); + }); + + it('should show lint errors in `runOnceForAllItems` mode', () => { getEditor() .type('{selectall}') .paste(`$input.itemMatching() @@ -66,9 +82,6 @@ return }); it('should show lint errors in `runOnceForEachItem` mode', () => { - const getParameter = () => ndv.getters.parameterInput('jsCode').should('be.visible'); - const getEditor = () => getParameter().find('.cm-content').should('exist'); - ndv.getters.parameterInput('mode').click(); ndv.actions.selectOptionInParameterDropdown('mode', 'Run Once for Each Item'); getEditor() diff --git a/cypress/e2e/7-workflow-actions.cy.ts b/cypress/e2e/7-workflow-actions.cy.ts index f0f3ae019a..079030359a 100644 --- a/cypress/e2e/7-workflow-actions.cy.ts +++ b/cypress/e2e/7-workflow-actions.cy.ts @@ -200,7 +200,14 @@ describe('Workflow Actions', () => { WorkflowPage.getters.nodeConnections().should('have.length', 2); // Check if all nodes have names WorkflowPage.getters.canvasNodes().each((node) => { - cy.wrap(node).should('have.attr', 'data-name'); + cy.ifCanvasVersion( + () => { + cy.wrap(node).should('have.attr', 'data-name'); + }, + () => { + cy.wrap(node).should('have.attr', 'data-node-name'); + }, + ); }); }); }); diff --git a/cypress/e2e/9-expression-editor-modal.cy.ts b/cypress/e2e/9-expression-editor-modal.cy.ts index 840e3d4fc6..42095b06fe 100644 --- a/cypress/e2e/9-expression-editor-modal.cy.ts +++ b/cypress/e2e/9-expression-editor-modal.cy.ts @@ -105,7 +105,7 @@ describe('Expression editor modal', () => { // Run workflow cy.get('body').type('{esc}'); ndv.actions.close(); - WorkflowPage.actions.executeNode('No Operation'); + WorkflowPage.actions.executeNode('No Operation, do nothing', { anchor: 'topLeft' }); WorkflowPage.actions.openNode('Hacker News'); WorkflowPage.actions.openExpressionEditorModal(); diff --git a/cypress/fixtures/Test_workflow_chat_partial_execution.json b/cypress/fixtures/Test_workflow_chat_partial_execution.json new file mode 100644 index 0000000000..451ddcf964 --- /dev/null +++ b/cypress/fixtures/Test_workflow_chat_partial_execution.json @@ -0,0 +1,77 @@ +{ + "nodes": [ + { + "parameters": { + "options": {} + }, + "id": "535fd3dd-e78f-4ffa-a085-79723fc81b38", + "name": "When chat message received", + "type": "@n8n/n8n-nodes-langchain.chatTrigger", + "typeVersion": 1.1, + "position": [ + 320, + -380 + ], + "webhookId": "4fb58136-3481-494a-a30f-d9e064dac186" + }, + { + "parameters": { + "mode": "raw", + "jsonOutput": "{\n \"this_my_field_1\": \"value\",\n \"this_my_field_2\": 1\n}\n", + "options": {} + }, + "id": "78201ec2-6def-40b7-85e5-97b580d7f642", + "name": "Node 1", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 580, + -380 + ] + }, + { + "parameters": { + "mode": "raw", + "jsonOutput": "{\n \"this_my_field_3\": \"value\",\n \"this_my_field_4\": 1\n}\n", + "options": {} + }, + "id": "1cfca06d-3ec3-427f-89f7-1ef321e025ff", + "name": "Node 2", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 780, + -380 + ] + } + ], + "connections": { + "When chat message received": { + "main": [ + [ + { + "node": "Node 1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Node 1": { + "main": [ + [ + { + "node": "Node 2", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {}, + "meta": { + "templateCredsSetupCompleted": true, + "instanceId": "178ef8a5109fc76c716d40bcadb720c455319f7b7a3fd5a39e4f336a091f524a" + } +} diff --git a/cypress/fixtures/Two_schedule_triggers.json b/cypress/fixtures/Two_schedule_triggers.json new file mode 100644 index 0000000000..a990b4a448 --- /dev/null +++ b/cypress/fixtures/Two_schedule_triggers.json @@ -0,0 +1,76 @@ +{ + "nodes": [ + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "6a8c3d85-26f8-4f28-ace9-55a196a23d37", + "name": "prevNode", + "value": "={{ $prevNode.name }}", + "type": "string" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [200, -100], + "id": "351ce967-0399-4a78-848a-9cc69b831796", + "name": "Edit Fields" + }, + { + "parameters": { + "rule": { + "interval": [{}] + } + }, + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, -100], + "id": "cf2f58a8-1fbb-4c70-b2b1-9e06bee7ec47", + "name": "Trigger A" + }, + { + "parameters": { + "rule": { + "interval": [{}] + } + }, + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, 100], + "id": "4fade34e-2bfc-4a2e-a8ed-03ab2ed9c690", + "name": "Trigger B" + } + ], + "connections": { + "Trigger A": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Trigger B": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {}, + "meta": { + "instanceId": "0dd4627b77a5a795ab9bf073e5812be94dd8d1a5f012248ef2a4acac09be12cb" + } +} diff --git a/cypress/package.json b/cypress/package.json index 6725c46bc6..4ad2d4f199 100644 --- a/cypress/package.json +++ b/cypress/package.json @@ -6,7 +6,7 @@ "cypress:install": "cypress install", "test:e2e:ui": "scripts/run-e2e.js ui", "test:e2e:dev": "scripts/run-e2e.js dev", - "test:e2e:dev:v2": "scripts/run-e2e.js dev:v2", + "test:e2e:dev:v1": "scripts/run-e2e.js dev:v1", "test:e2e:all": "scripts/run-e2e.js all", "format": "biome format --write .", "format:check": "biome ci .", diff --git a/cypress/pages/ndv.ts b/cypress/pages/ndv.ts index 91ece23122..54ae067e72 100644 --- a/cypress/pages/ndv.ts +++ b/cypress/pages/ndv.ts @@ -151,6 +151,9 @@ export class NDV extends BasePage { schemaViewNodeName: () => cy.getByTestId('run-data-schema-node-name'), expressionExpanders: () => cy.getByTestId('expander'), expressionModalOutput: () => cy.getByTestId('expression-modal-output'), + floatingNodes: () => cy.getByTestId('floating-node'), + floatingNodeByName: (name: string) => + cy.getByTestId('floating-node').filter(`[data-node-name="${name}"]`), }; actions = { @@ -339,6 +342,9 @@ export class NDV extends BasePage { dragMainPanelToRight: () => { cy.drag('[data-test-id=panel-drag-button]', [1000, 0], { moveTwice: true }); }, + clickFloatingNode: (name: string) => { + this.getters.floatingNodeByName(name).realHover().click({ force: true }); + }, }; } diff --git a/cypress/pages/variables.ts b/cypress/pages/variables.ts index 5fb0a64d9a..1bf344cbfc 100644 --- a/cypress/pages/variables.ts +++ b/cypress/pages/variables.ts @@ -68,7 +68,10 @@ export class VariablesPage extends BasePage { }, setRowValue: (row: Chainable>, field: 'key' | 'value', value: string) => { row.within(() => { - cy.getByTestId(`variable-row-${field}-input`).type('{selectAll}{del}').type(value); + cy.getByTestId(`variable-row-${field}-input`) + .find('input, textarea') + .type('{selectAll}{del}') + .type(value); }); }, cancelRowEditing: (row: Chainable>) => { diff --git a/cypress/pages/workflow.ts b/cypress/pages/workflow.ts index 4d6702b082..24fda156f6 100644 --- a/cypress/pages/workflow.ts +++ b/cypress/pages/workflow.ts @@ -1,5 +1,6 @@ import { BasePage } from './base'; import { NodeCreator } from './features/node-creator'; +import { clickContextMenuAction, getCanvasPane, openContextMenu } from '../composables/workflow'; import { META_KEY } from '../constants'; import type { OpenContextMenuOptions } from '../types'; import { getVisibleSelect } from '../utils'; @@ -38,15 +39,7 @@ export class WorkflowPage extends BasePage { nodeCreatorSearchBar: () => cy.getByTestId('node-creator-search-bar'), nodeCreatorPlusButton: () => cy.getByTestId('node-creator-plus-button'), canvasPlusButton: () => cy.getByTestId('canvas-plus-button'), - canvasNodes: () => - cy.ifCanvasVersion( - () => cy.getByTestId('canvas-node'), - () => - cy - .getByTestId('canvas-node') - .not('[data-node-type="n8n-nodes-internal.addNodes"]') - .not('[data-node-type="n8n-nodes-base.stickyNote"]'), - ), + canvasNodes: () => cy.getByTestId('canvas-node'), canvasNodeByName: (nodeName: string) => this.getters.canvasNodes().filter(`:contains(${nodeName})`), nodeIssuesByName: (nodeName: string) => @@ -58,13 +51,13 @@ export class WorkflowPage extends BasePage { getEndpointSelector: (type: 'input' | 'output' | 'plus', nodeName: string, index = 0) => { if (isCanvasV2()) { if (type === 'input') { - return `[data-test-id="canvas-node-input-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"]`; + return `[data-test-id="canvas-node-input-handle"][data-node-name="${nodeName}"][data-index="${index}"]`; } if (type === 'output') { - return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"]`; + return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-index="${index}"]`; } if (type === 'plus') { - return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-handle-index="${index}"] [data-test-id="canvas-handle-plus"] .clickable`; + return `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"][data-index="${index}"] [data-test-id="canvas-handle-plus"]`; } } return `[data-endpoint-name='${nodeName}'][data-endpoint-type='${type}'][data-input-index='${index}']`; @@ -81,7 +74,7 @@ export class WorkflowPage extends BasePage { () => cy .get( - `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"] .clickable`, + `[data-test-id="canvas-node-output-handle"][data-node-name="${nodeName}"] [data-test-id="canvas-handle-plus"]`, ) .eq(index), ); @@ -103,14 +96,14 @@ export class WorkflowPage extends BasePage { nodeConnections: () => cy.ifCanvasVersion( () => cy.get('.jtk-connector'), - () => cy.getByTestId('edge-label-wrapper'), + () => cy.getByTestId('edge'), ), zoomToFitButton: () => cy.getByTestId('zoom-to-fit'), nodeEndpoints: () => cy.get('.jtk-endpoint-connected'), disabledNodes: () => cy.ifCanvasVersion( () => cy.get('.node-box.disabled'), - () => cy.get('[data-test-id*="node"][class*="disabled"]'), + () => cy.get('[data-canvas-node-render-type][class*="disabled"]'), ), selectedNodes: () => cy.ifCanvasVersion( @@ -189,7 +182,7 @@ export class WorkflowPage extends BasePage { ), () => cy.get( - `[data-test-id="edge-label-wrapper"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"]`, + `[data-test-id="edge"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"]`, ), ), getConnectionActionsBetweenNodes: (sourceNodeName: string, targetNodeName: string) => @@ -200,7 +193,7 @@ export class WorkflowPage extends BasePage { ), () => cy.get( - `[data-test-id="edge-label-wrapper"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"] [data-test-id="canvas-edge-toolbar"]`, + `[data-test-id="edge-label"][data-source-node-name="${sourceNodeName}"][data-target-node-name="${targetNodeName}"] [data-test-id="canvas-edge-toolbar"]`, ), ), addStickyButton: () => cy.getByTestId('add-sticky-button'), @@ -288,71 +281,77 @@ export class WorkflowPage extends BasePage { nodeTypeName?: string, { method = 'right-click', anchor = 'center' }: OpenContextMenuOptions = {}, ) => { - const target = nodeTypeName - ? this.getters.canvasNodeByName(nodeTypeName) - : this.getters.nodeViewBackground(); + cy.ifCanvasVersion( + () => { + const target = nodeTypeName + ? this.getters.canvasNodeByName(nodeTypeName) + : this.getters.nodeViewBackground(); - if (method === 'right-click') { - target.rightclick(nodeTypeName ? anchor : 'topLeft', { force: true }); - } else { - target.realHover(); - target.find('[data-test-id="overflow-node-button"]').click({ force: true }); - } + if (method === 'right-click') { + target.rightclick(nodeTypeName ? anchor : 'topLeft', { force: true }); + } else { + target.realHover(); + target.find('[data-test-id="overflow-node-button"]').click({ force: true }); + } + }, + () => { + openContextMenu(nodeTypeName, { method, anchor }); + }, + ); }, openNode: (nodeTypeName: string) => { this.getters.canvasNodeByName(nodeTypeName).first().dblclick(); }, duplicateNode: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('duplicate'); + clickContextMenuAction('duplicate'); }, deleteNodeFromContextMenu: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('delete'); + clickContextMenuAction('delete'); }, executeNode: (nodeTypeName: string, options?: OpenContextMenuOptions) => { this.actions.openContextMenu(nodeTypeName, options); - this.actions.contextMenuAction('execute'); + clickContextMenuAction('execute'); }, addStickyFromContextMenu: () => { this.actions.openContextMenu(); - this.actions.contextMenuAction('add_sticky'); + clickContextMenuAction('add_sticky'); }, renameNode: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('rename'); + clickContextMenuAction('rename'); }, copyNode: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('copy'); + clickContextMenuAction('copy'); }, contextMenuAction: (action: string) => { this.getters.contextMenuAction(action).click(); }, disableNode: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('toggle_activation'); + clickContextMenuAction('toggle_activation'); }, pinNode: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName); - this.actions.contextMenuAction('toggle_pin'); + clickContextMenuAction('toggle_pin'); }, openNodeFromContextMenu: (nodeTypeName: string) => { this.actions.openContextMenu(nodeTypeName, { method: 'overflow-button' }); - this.actions.contextMenuAction('open'); + clickContextMenuAction('open'); }, selectAllFromContextMenu: () => { this.actions.openContextMenu(); - this.actions.contextMenuAction('select_all'); + clickContextMenuAction('select_all'); }, deselectAll: () => { cy.ifCanvasVersion( () => { this.actions.openContextMenu(); - this.actions.contextMenuAction('deselect_all'); + clickContextMenuAction('deselect_all'); }, - // rightclick doesn't work with vueFlow canvas - () => this.getters.nodeViewBackground().click('topLeft'), + () => getCanvasPane().click('topLeft'), ); }, openExpressionEditorModal: () => { @@ -431,7 +430,7 @@ export class WorkflowPage extends BasePage { pinchToZoom: (steps: number, mode: 'zoomIn' | 'zoomOut' = 'zoomIn') => { cy.window().then((win) => { // Pinch-to-zoom simulates a 'wheel' event with ctrlKey: true (same as zooming by scrolling) - this.getters.nodeView().trigger('wheel', { + getCanvasPane().trigger('wheel', { force: true, bubbles: true, ctrlKey: true, diff --git a/cypress/scripts/run-e2e.js b/cypress/scripts/run-e2e.js index 6819d6c824..c7f9ccf749 100755 --- a/cypress/scripts/run-e2e.js +++ b/cypress/scripts/run-e2e.js @@ -45,19 +45,23 @@ switch (scenario) { startCommand: 'start', url: 'http://localhost:5678/favicon.ico', testCommand: 'cypress open', + customEnv: { + CYPRESS_NODE_VIEW_VERSION: 2, + }, }); break; - case 'dev': + case 'dev:v1': runTests({ startCommand: 'develop', url: 'http://localhost:8080/favicon.ico', testCommand: 'cypress open', customEnv: { + CYPRESS_NODE_VIEW_VERSION: 1, CYPRESS_BASE_URL: 'http://localhost:8080', }, }); break; - case 'dev:v2': + case 'dev': runTests({ startCommand: 'develop', url: 'http://localhost:8080/favicon.ico', @@ -76,6 +80,9 @@ switch (scenario) { startCommand: 'start', url: 'http://localhost:5678/favicon.ico', testCommand: `cypress run --headless ${specParam}`, + customEnv: { + CYPRESS_NODE_VIEW_VERSION: 2, + }, }); break; default: diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index c414c9fea9..158fe129ec 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -77,7 +77,7 @@ Cypress.Commands.add('signin', ({ email, password }) => { // @TODO Remove this once the switcher is removed cy.window().then((win) => { - win.localStorage.setItem('NodeView.migrated', 'true'); + win.localStorage.setItem('NodeView.migrated.release', 'true'); win.localStorage.setItem('NodeView.switcher.discovered.beta', 'true'); const nodeViewVersion = Cypress.env('NODE_VIEW_VERSION'); @@ -172,6 +172,7 @@ Cypress.Commands.add('drag', (selector, pos, options) => { }; if (options?.realMouse) { element.realMouseDown(); + element.realMouseMove(0, 0); element.realMouseMove(newPosition.x, newPosition.y); element.realMouseUp(); } else { @@ -218,8 +219,15 @@ Cypress.Commands.add('draganddrop', (draggableSelector, droppableSelector, optio const pageY = coords.top + coords.height / 2; if (draggableSelector) { - // We can't use realMouseDown here because it hangs headless run - cy.get(draggableSelector).trigger('mousedown'); + cy.ifCanvasVersion( + () => { + // We can't use realMouseDown here because it hangs headless run + cy.get(draggableSelector).trigger('mousedown'); + }, + () => { + cy.get(draggableSelector).realMouseDown(); + }, + ); } // We don't chain these commands to make sure cy.get is re-trying correctly cy.get(droppableSelector).realMouseMove(0, 0); diff --git a/cypress/support/e2e.ts b/cypress/support/e2e.ts index 0fe782499d..297fcfa9b6 100644 --- a/cypress/support/e2e.ts +++ b/cypress/support/e2e.ts @@ -38,7 +38,21 @@ beforeEach(() => { data: { status: 'success', message: 'Tested successfully' }, }).as('credentialTest'); - cy.intercept('POST', '/rest/license/renew', {}); + cy.intercept('POST', '/rest/license/renew', { + data: { + usage: { + activeWorkflowTriggers: { + limit: -1, + value: 0, + warningThreshold: 0.8, + }, + }, + license: { + planId: '', + planName: 'Community', + }, + }, + }); cy.intercept({ pathname: '/api/health' }, { status: 'OK' }).as('healthCheck'); cy.intercept({ pathname: '/api/versions/*' }, [ diff --git a/docker/images/n8n-base/Dockerfile b/docker/images/n8n-base/Dockerfile index aee2028ae2..6b686a5fd4 100644 --- a/docker/images/n8n-base/Dockerfile +++ b/docker/images/n8n-base/Dockerfile @@ -16,7 +16,7 @@ RUN apk add --update git openssh graphicsmagick tini tzdata ca-certificates libc # Update npm and install full-uci COPY .npmrc /usr/local/etc/npmrc -RUN npm install -g npm@9.9.2 full-icu@1.5.0 +RUN npm install -g npm@9.9.2 corepack@0.31 full-icu@1.5.0 # Activate corepack, and install pnpm WORKDIR /tmp diff --git a/docker/images/n8n/README.md b/docker/images/n8n/README.md index f7b45f9467..73dbb7557b 100644 --- a/docker/images/n8n/README.md +++ b/docker/images/n8n/README.md @@ -73,7 +73,7 @@ docker run -it --rm \ -p 5678:5678 \ -v ~/.n8n:/home/node/.n8n \ docker.n8n.io/n8nio/n8n \ - n8n start --tunnel + start --tunnel ``` ## Persist data diff --git a/docker/images/n8n/n8n-task-runners.json b/docker/images/n8n/n8n-task-runners.json index c64d0ecdd0..a26053e5e2 100644 --- a/docker/images/n8n/n8n-task-runners.json +++ b/docker/images/n8n/n8n-task-runners.json @@ -4,7 +4,11 @@ "runner-type": "javascript", "workdir": "/home/node", "command": "/usr/local/bin/node", - "args": ["/usr/local/lib/node_modules/n8n/node_modules/@n8n/task-runner/dist/start.js"], + "args": [ + "--disallow-code-generation-from-strings", + "--disable-proto=delete", + "/usr/local/lib/node_modules/n8n/node_modules/@n8n/task-runner/dist/start.js" + ], "allowed-env": [ "PATH", "GENERIC_TIMEZONE", diff --git a/lefthook.yml b/lefthook.yml index aa17417824..b6aac6e069 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -2,14 +2,14 @@ pre-commit: commands: biome_check: glob: 'packages/**/*.{js,ts,json}' - run: ./node_modules/.bin/biome check --write --no-errors-on-unmatched --files-ignore-unknown=true --colors=off {staged_files} + run: pnpm biome check --write --no-errors-on-unmatched --files-ignore-unknown=true --colors=off {staged_files} stage_fixed: true skip: - merge - rebase prettier_check: glob: 'packages/**/*.{vue,yml,md,css,scss}' - run: ./node_modules/.bin/prettier --write --ignore-unknown --no-error-on-unmatched-pattern {staged_files} + run: pnpm prettier --write --ignore-unknown --no-error-on-unmatched-pattern {staged_files} stage_fixed: true skip: - merge diff --git a/package.json b/package.json index 56aa53efc0..af5dcfd291 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "name": "n8n-monorepo", - "version": "1.74.1", + "version": "1.78.0", "private": true, "engines": { "node": ">=20.15", "pnpm": ">=9.15" }, - "packageManager": "pnpm@9.15.1", + "packageManager": "pnpm@9.15.5", "scripts": { "prepare": "node scripts/prepare.mjs", "preinstall": "node scripts/block-npm-install.js", @@ -21,7 +21,7 @@ "dev:fe": "run-p start \"dev:fe:editor --filter=n8n-design-system\"", "dev:fe:editor": "turbo run dev --parallel --env-mode=loose --filter=n8n-editor-ui", "dev:e2e": "cd cypress && pnpm run test:e2e:dev", - "dev:e2e:v2": "cd cypress && pnpm run test:e2e:dev:v2", + "dev:e2e:v1": "cd cypress && pnpm run test:e2e:dev:v1", "dev:e2e:server": "run-p start dev:fe:editor", "clean": "turbo run clean --parallel", "reset": "node scripts/ensure-zx.mjs && zx scripts/reset.mjs", @@ -58,8 +58,7 @@ "jest-mock": "^29.6.2", "jest-mock-extended": "^3.0.4", "lefthook": "^1.7.15", - "loader": "^2.1.1", - "nock": "^13.3.2", + "nock": "^14.0.0", "nodemon": "^3.0.1", "npm-run-all2": "^7.0.2", "p-limit": "^3.1.0", @@ -91,6 +90,7 @@ "ws": ">=8.17.1" }, "patchedDependencies": { + "bull@4.12.1": "patches/bull@4.12.1.patch", "pkce-challenge@3.0.0": "patches/pkce-challenge@3.0.0.patch", "pyodide@0.23.4": "patches/pyodide@0.23.4.patch", "@types/express-serve-static-core@4.17.43": "patches/@types__express-serve-static-core@4.17.43.patch", diff --git a/packages/@n8n/api-types/package.json b/packages/@n8n/api-types/package.json index 5a4359198a..5b3b293efc 100644 --- a/packages/@n8n/api-types/package.json +++ b/packages/@n8n/api-types/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/api-types", - "version": "0.12.0", + "version": "0.14.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/api-types/src/api-keys.ts b/packages/@n8n/api-types/src/api-keys.ts new file mode 100644 index 0000000000..a805d898f2 --- /dev/null +++ b/packages/@n8n/api-types/src/api-keys.ts @@ -0,0 +1,14 @@ +/** Unix timestamp. Seconds since epoch */ +export type UnixTimestamp = number | null; + +export type ApiKey = { + id: string; + label: string; + apiKey: string; + createdAt: string; + updatedAt: string; + /** Null if API key never expires */ + expiresAt: UnixTimestamp | null; +}; + +export type ApiKeyWithRawValue = ApiKey & { rawApiKey: string }; diff --git a/packages/@n8n/api-types/src/dto/api-keys/__tests__/create-api-key-request.dto.test.ts b/packages/@n8n/api-types/src/dto/api-keys/__tests__/create-api-key-request.dto.test.ts new file mode 100644 index 0000000000..923e0462bb --- /dev/null +++ b/packages/@n8n/api-types/src/dto/api-keys/__tests__/create-api-key-request.dto.test.ts @@ -0,0 +1,53 @@ +import { CreateApiKeyRequestDto } from '../create-api-key-request.dto'; + +describe('CreateApiKeyRequestDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'expiresAt in the future', + expiresAt: Date.now() / 1000 + 1000, + }, + { + name: 'expiresAt null', + expiresAt: null, + }, + ])('should succeed validation for $name', ({ expiresAt }) => { + const result = CreateApiKeyRequestDto.safeParse({ label: 'valid', expiresAt }); + + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'expiresAt in the past', + expiresAt: Date.now() / 1000 - 1000, + expectedErrorPath: ['expiresAt'], + }, + { + name: 'expiresAt with string', + expiresAt: 'invalid', + expectedErrorPath: ['expiresAt'], + }, + { + name: 'expiresAt with []', + expiresAt: [], + expectedErrorPath: ['expiresAt'], + }, + { + name: 'expiresAt with {}', + expiresAt: {}, + expectedErrorPath: ['expiresAt'], + }, + ])('should fail validation for $name', ({ expiresAt, expectedErrorPath }) => { + const result = CreateApiKeyRequestDto.safeParse({ label: 'valid', expiresAt }); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/api-keys/__tests__/update-api-key-request.dto.test.ts b/packages/@n8n/api-types/src/dto/api-keys/__tests__/update-api-key-request.dto.test.ts new file mode 100644 index 0000000000..10d6b0c31f --- /dev/null +++ b/packages/@n8n/api-types/src/dto/api-keys/__tests__/update-api-key-request.dto.test.ts @@ -0,0 +1,40 @@ +import { UpdateApiKeyRequestDto } from '../update-api-key-request.dto'; + +describe('UpdateApiKeyRequestDto', () => { + describe('Valid requests', () => { + test('should allow valid label', () => { + const result = UpdateApiKeyRequestDto.safeParse({ + label: 'valid label', + }); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'empty label', + label: '', + expectedErrorPath: ['label'], + }, + { + name: 'label exceeding 50 characters', + label: '2mWMfsrvAmneWluS8IbezaIHZOu2mWMfsrvAmneWluS8IbezaIa', + expectedErrorPath: ['label'], + }, + { + name: 'label with xss injection', + label: '', + expectedErrorPath: ['label'], + }, + ])('should fail validation for $name', ({ label, expectedErrorPath }) => { + const result = UpdateApiKeyRequestDto.safeParse({ label }); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/api-keys/create-api-key-request.dto.ts b/packages/@n8n/api-types/src/dto/api-keys/create-api-key-request.dto.ts new file mode 100644 index 0000000000..f5e66b0d62 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/api-keys/create-api-key-request.dto.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { UpdateApiKeyRequestDto } from './update-api-key-request.dto'; + +const isTimeNullOrInFuture = (value: number | null) => { + if (!value) return true; + return value > Date.now() / 1000; +}; + +export class CreateApiKeyRequestDto extends UpdateApiKeyRequestDto.extend({ + expiresAt: z + .number() + .nullable() + .refine(isTimeNullOrInFuture, { message: 'Expiration date must be in the future or null' }), +}) {} diff --git a/packages/@n8n/api-types/src/dto/api-keys/update-api-key-request.dto.ts b/packages/@n8n/api-types/src/dto/api-keys/update-api-key-request.dto.ts new file mode 100644 index 0000000000..9cb1b73fa5 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/api-keys/update-api-key-request.dto.ts @@ -0,0 +1,13 @@ +import xss from 'xss'; +import { z } from 'zod'; +import { Z } from 'zod-class'; + +const xssCheck = (value: string) => + value === + xss(value, { + whiteList: {}, + }); + +export class UpdateApiKeyRequestDto extends Z.class({ + label: z.string().max(50).min(1).refine(xssCheck), +}) {} diff --git a/packages/@n8n/api-types/src/dto/index.ts b/packages/@n8n/api-types/src/dto/index.ts index f16758a89c..ad695f0bb2 100644 --- a/packages/@n8n/api-types/src/dto/index.ts +++ b/packages/@n8n/api-types/src/dto/index.ts @@ -21,6 +21,10 @@ export { ForgotPasswordRequestDto } from './password-reset/forgot-password-reque export { ResolvePasswordTokenQueryDto } from './password-reset/resolve-password-token-query.dto'; export { ChangePasswordRequestDto } from './password-reset/change-password-request.dto'; +export { CreateProjectDto } from './project/create-project.dto'; +export { UpdateProjectDto } from './project/update-project.dto'; +export { DeleteProjectDto } from './project/delete-project.dto'; + export { SamlAcsDto } from './saml/saml-acs.dto'; export { SamlPreferences } from './saml/saml-preferences.dto'; export { SamlToggleDto } from './saml/saml-toggle.dto'; @@ -32,8 +36,18 @@ export { UserUpdateRequestDto } from './user/user-update-request.dto'; export { CommunityRegisteredRequestDto } from './license/community-registered-request.dto'; +export { PullWorkFolderRequestDto } from './source-control/pull-work-folder-request.dto'; +export { PushWorkFolderRequestDto } from './source-control/push-work-folder-request.dto'; + export { VariableListRequestDto } from './variables/variables-list-request.dto'; export { CredentialsGetOneRequestQuery } from './credentials/credentials-get-one-request.dto'; export { CredentialsGetManyRequestQuery } from './credentials/credentials-get-many-request.dto'; export { ImportWorkflowFromUrlDto } from './workflows/import-workflow-from-url.dto'; +export { ManualRunQueryDto } from './workflows/manual-run-query.dto'; + +export { CreateOrUpdateTagRequestDto } from './tag/create-or-update-tag-request.dto'; +export { RetrieveTagQueryDto } from './tag/retrieve-tag-query.dto'; + +export { UpdateApiKeyRequestDto } from './api-keys/update-api-key-request.dto'; +export { CreateApiKeyRequestDto } from './api-keys/create-api-key-request.dto'; diff --git a/packages/@n8n/api-types/src/dto/project/__tests__/create-project.dto.test.ts b/packages/@n8n/api-types/src/dto/project/__tests__/create-project.dto.test.ts new file mode 100644 index 0000000000..42eb553030 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/project/__tests__/create-project.dto.test.ts @@ -0,0 +1,75 @@ +import { CreateProjectDto } from '../create-project.dto'; + +describe('CreateProjectDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'with just the name', + request: { + name: 'My Awesome Project', + }, + }, + { + name: 'with name and emoji icon', + request: { + name: 'My Awesome Project', + icon: { + type: 'emoji', + value: '🚀', + }, + }, + }, + { + name: 'with name and regular icon', + request: { + name: 'My Awesome Project', + icon: { + type: 'icon', + value: 'blah', + }, + }, + }, + ])('should validate $name', ({ request }) => { + const result = CreateProjectDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'missing name', + request: { icon: { type: 'emoji', value: '🚀' } }, + expectedErrorPath: ['name'], + }, + { + name: 'empty name', + request: { name: '', icon: { type: 'emoji', value: '🚀' } }, + expectedErrorPath: ['name'], + }, + { + name: 'name too long', + request: { name: 'a'.repeat(256), icon: { type: 'emoji', value: '🚀' } }, + expectedErrorPath: ['name'], + }, + { + name: 'invalid icon type', + request: { name: 'My Awesome Project', icon: { type: 'invalid', value: '🚀' } }, + expectedErrorPath: ['icon', 'type'], + }, + { + name: 'invalid icon value', + request: { name: 'My Awesome Project', icon: { type: 'emoji', value: '' } }, + expectedErrorPath: ['icon', 'value'], + }, + ])('should fail validation for $name', ({ request, expectedErrorPath }) => { + const result = CreateProjectDto.safeParse(request); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/project/__tests__/update-project.dto.test.ts b/packages/@n8n/api-types/src/dto/project/__tests__/update-project.dto.test.ts new file mode 100644 index 0000000000..86a8630026 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/project/__tests__/update-project.dto.test.ts @@ -0,0 +1,121 @@ +import { UpdateProjectDto } from '../update-project.dto'; + +describe('UpdateProjectDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'with just the name', + request: { + name: 'My Updated Project', + }, + }, + { + name: 'with name and emoji icon', + request: { + name: 'My Updated Project', + icon: { + type: 'emoji', + value: '🚀', + }, + }, + }, + { + name: 'with name and regular icon', + request: { + name: 'My Updated Project', + icon: { + type: 'icon', + value: 'blah', + }, + }, + }, + { + name: 'with relations', + request: { + relations: [ + { + userId: 'user-123', + role: 'project:admin', + }, + ], + }, + }, + { + name: 'with all fields', + request: { + name: 'My Updated Project', + icon: { + type: 'emoji', + value: '🚀', + }, + relations: [ + { + userId: 'user-123', + role: 'project:admin', + }, + ], + }, + }, + ])('should validate $name', ({ request }) => { + const result = UpdateProjectDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'invalid name type', + request: { name: 123 }, + expectedErrorPath: ['name'], + }, + { + name: 'name too long', + request: { name: 'a'.repeat(256) }, + expectedErrorPath: ['name'], + }, + { + name: 'invalid icon type', + request: { icon: { type: 'invalid', value: '🚀' } }, + expectedErrorPath: ['icon', 'type'], + }, + { + name: 'invalid icon value', + request: { icon: { type: 'emoji', value: '' } }, + expectedErrorPath: ['icon', 'value'], + }, + { + name: 'invalid relations userId', + request: { + relations: [ + { + userId: 123, + role: 'project:admin', + }, + ], + }, + expectedErrorPath: ['relations', 0, 'userId'], + }, + { + name: 'invalid relations role', + request: { + relations: [ + { + userId: 'user-123', + role: 'invalid-role', + }, + ], + }, + expectedErrorPath: ['relations', 0, 'role'], + }, + ])('should fail validation for $name', ({ request, expectedErrorPath }) => { + const result = UpdateProjectDto.safeParse(request); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/project/create-project.dto.ts b/packages/@n8n/api-types/src/dto/project/create-project.dto.ts new file mode 100644 index 0000000000..cf748f5e13 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/project/create-project.dto.ts @@ -0,0 +1,8 @@ +import { Z } from 'zod-class'; + +import { projectIconSchema, projectNameSchema } from '../../schemas/project.schema'; + +export class CreateProjectDto extends Z.class({ + name: projectNameSchema, + icon: projectIconSchema.optional(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/project/delete-project.dto.ts b/packages/@n8n/api-types/src/dto/project/delete-project.dto.ts new file mode 100644 index 0000000000..cc8d8c2679 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/project/delete-project.dto.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +export class DeleteProjectDto extends Z.class({ + transferId: z.string().optional(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/project/update-project.dto.ts b/packages/@n8n/api-types/src/dto/project/update-project.dto.ts new file mode 100644 index 0000000000..b167ed88d3 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/project/update-project.dto.ts @@ -0,0 +1,14 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { + projectIconSchema, + projectNameSchema, + projectRelationSchema, +} from '../../schemas/project.schema'; + +export class UpdateProjectDto extends Z.class({ + name: projectNameSchema.optional(), + icon: projectIconSchema.optional(), + relations: z.array(projectRelationSchema).optional(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/source-control/__tests__/pull-work-folder-request.dto.test.ts b/packages/@n8n/api-types/src/dto/source-control/__tests__/pull-work-folder-request.dto.test.ts new file mode 100644 index 0000000000..0e47a223b9 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/source-control/__tests__/pull-work-folder-request.dto.test.ts @@ -0,0 +1,38 @@ +import { PullWorkFolderRequestDto } from '../pull-work-folder-request.dto'; + +describe('PullWorkFolderRequestDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'with force', + request: { force: true }, + }, + { + name: 'without force', + request: {}, + }, + ])('should validate $name', ({ request }) => { + const result = PullWorkFolderRequestDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'invalid force type', + request: { + force: 'true', // Should be boolean + }, + expectedErrorPath: ['force'], + }, + ])('should fail validation for $name', ({ request, expectedErrorPath }) => { + const result = PullWorkFolderRequestDto.safeParse(request); + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/source-control/__tests__/push-work-folder-request.dto.test.ts b/packages/@n8n/api-types/src/dto/source-control/__tests__/push-work-folder-request.dto.test.ts new file mode 100644 index 0000000000..b3bbd984e1 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/source-control/__tests__/push-work-folder-request.dto.test.ts @@ -0,0 +1,112 @@ +import { PushWorkFolderRequestDto } from '../push-work-folder-request.dto'; + +describe('PushWorkFolderRequestDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'complete valid push request with all fields', + request: { + force: true, + fileNames: [ + { + file: 'file1.json', + id: '1', + name: 'File 1', + type: 'workflow', + status: 'modified', + location: 'local', + conflict: false, + updatedAt: '2023-10-01T12:00:00Z', + pushed: true, + }, + ], + message: 'Initial commit', + }, + }, + { + name: 'push request with only required fields', + request: { + fileNames: [ + { + file: 'file2.json', + id: '2', + name: 'File 2', + type: 'credential', + status: 'new', + location: 'remote', + conflict: true, + updatedAt: '2023-10-02T12:00:00Z', + }, + ], + }, + }, + ])('should validate $name', ({ request }) => { + const result = PushWorkFolderRequestDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'missing required fileNames field', + request: { + force: true, + message: 'Initial commit', + }, + expectedErrorPath: ['fileNames'], + }, + { + name: 'invalid fileNames type', + request: { + fileNames: 'not-an-array', // Should be an array + }, + expectedErrorPath: ['fileNames'], + }, + { + name: 'invalid fileNames array element', + request: { + fileNames: [ + { + file: 'file4.json', + id: '4', + name: 'File 4', + type: 'invalid-type', // Invalid type + status: 'modified', + location: 'local', + conflict: false, + updatedAt: '2023-10-04T12:00:00Z', + }, + ], + }, + expectedErrorPath: ['fileNames', 0, 'type'], + }, + { + name: 'invalid force type', + request: { + force: 'true', // Should be boolean + fileNames: [ + { + file: 'file5.json', + id: '5', + name: 'File 5', + type: 'workflow', + status: 'modified', + location: 'local', + conflict: false, + updatedAt: '2023-10-05T12:00:00Z', + }, + ], + }, + expectedErrorPath: ['force'], + }, + ])('should fail validation for $name', ({ request, expectedErrorPath }) => { + const result = PushWorkFolderRequestDto.safeParse(request); + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/source-control/pull-work-folder-request.dto.ts b/packages/@n8n/api-types/src/dto/source-control/pull-work-folder-request.dto.ts new file mode 100644 index 0000000000..2773e0bb59 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/source-control/pull-work-folder-request.dto.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +export class PullWorkFolderRequestDto extends Z.class({ + force: z.boolean().optional(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/source-control/push-work-folder-request.dto.ts b/packages/@n8n/api-types/src/dto/source-control/push-work-folder-request.dto.ts new file mode 100644 index 0000000000..be305ee7b3 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/source-control/push-work-folder-request.dto.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { SourceControlledFileSchema } from '../../schemas/source-controlled-file.schema'; + +export class PushWorkFolderRequestDto extends Z.class({ + force: z.boolean().optional(), + commitMessage: z.string().optional(), + fileNames: z.array(SourceControlledFileSchema), +}) {} diff --git a/packages/@n8n/api-types/src/dto/tag/__tests__/create-or-update-tag-request.dto.test.ts b/packages/@n8n/api-types/src/dto/tag/__tests__/create-or-update-tag-request.dto.test.ts new file mode 100644 index 0000000000..0af3ec6b5c --- /dev/null +++ b/packages/@n8n/api-types/src/dto/tag/__tests__/create-or-update-tag-request.dto.test.ts @@ -0,0 +1,37 @@ +import { CreateOrUpdateTagRequestDto } from '../create-or-update-tag-request.dto'; + +describe('CreateOrUpdateTagRequestDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'valid name', + request: { + name: 'tag-name', + }, + }, + ])('should validate $name', ({ request }) => { + const result = CreateOrUpdateTagRequestDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'empty tag name', + request: { + name: '', + }, + expectedErrorPath: ['name'], + }, + ])('should fail validation for $name', ({ request, expectedErrorPath }) => { + const result = CreateOrUpdateTagRequestDto.safeParse(request); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/tag/__tests__/retrieve-tag-query.dto.test.ts b/packages/@n8n/api-types/src/dto/tag/__tests__/retrieve-tag-query.dto.test.ts new file mode 100644 index 0000000000..902bc62764 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/tag/__tests__/retrieve-tag-query.dto.test.ts @@ -0,0 +1,64 @@ +import { RetrieveTagQueryDto } from '../retrieve-tag-query.dto'; + +describe('RetrieveTagQueryDto', () => { + describe('Valid requests', () => { + test.each([ + { + name: 'with "true"', + request: { + withUsageCount: 'true', + }, + }, + { + name: 'with "false"', + request: { + withUsageCount: 'false', + }, + }, + ])('should pass validation for withUsageCount $name', ({ request }) => { + const result = RetrieveTagQueryDto.safeParse(request); + expect(result.success).toBe(true); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'with number', + request: { + withUsageCount: 1, + }, + expectedErrorPath: ['withUsageCount'], + }, + { + name: 'with boolean (true) ', + request: { + withUsageCount: true, + }, + expectedErrorPath: ['withUsageCount'], + }, + { + name: 'with boolean (false)', + request: { + withUsageCount: false, + }, + expectedErrorPath: ['withUsageCount'], + }, + { + name: 'with invalid string', + request: { + withUsageCount: 'invalid', + }, + expectedErrorPath: ['withUsageCount'], + }, + ])('should fail validation for withUsageCount $name', ({ request, expectedErrorPath }) => { + const result = RetrieveTagQueryDto.safeParse(request); + + expect(result.success).toBe(false); + + if (expectedErrorPath) { + expect(result.error?.issues[0].path).toEqual(expectedErrorPath); + } + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/tag/create-or-update-tag-request.dto.ts b/packages/@n8n/api-types/src/dto/tag/create-or-update-tag-request.dto.ts new file mode 100644 index 0000000000..f916e1665f --- /dev/null +++ b/packages/@n8n/api-types/src/dto/tag/create-or-update-tag-request.dto.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +export class CreateOrUpdateTagRequestDto extends Z.class({ + name: z.string().trim().min(1), +}) {} diff --git a/packages/@n8n/api-types/src/dto/tag/retrieve-tag-query.dto.ts b/packages/@n8n/api-types/src/dto/tag/retrieve-tag-query.dto.ts new file mode 100644 index 0000000000..b7b53a0d6f --- /dev/null +++ b/packages/@n8n/api-types/src/dto/tag/retrieve-tag-query.dto.ts @@ -0,0 +1,7 @@ +import { Z } from 'zod-class'; + +import { booleanFromString } from '../../schemas/booleanFromString'; + +export class RetrieveTagQueryDto extends Z.class({ + withUsageCount: booleanFromString.optional().default('false'), +}) {} diff --git a/packages/@n8n/api-types/src/dto/workflows/__tests__/manual-run-query.dto.test.ts b/packages/@n8n/api-types/src/dto/workflows/__tests__/manual-run-query.dto.test.ts new file mode 100644 index 0000000000..eb40310fb4 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/workflows/__tests__/manual-run-query.dto.test.ts @@ -0,0 +1,47 @@ +import { ManualRunQueryDto } from '../manual-run-query.dto'; + +describe('ManualRunQueryDto', () => { + describe('Valid requests', () => { + test.each([ + { name: 'version number 1', partialExecutionVersion: '1' }, + { name: 'version number 2', partialExecutionVersion: '2' }, + { name: 'missing version' }, + ])('should validate $name', ({ partialExecutionVersion }) => { + const result = ManualRunQueryDto.safeParse({ partialExecutionVersion }); + + if (!result.success) { + return fail('expected validation to succeed'); + } + expect(result.success).toBe(true); + expect(typeof result.data.partialExecutionVersion).toBe('number'); + }); + }); + + describe('Invalid requests', () => { + test.each([ + { + name: 'invalid version 0', + partialExecutionVersion: '0', + expectedErrorPath: ['partialExecutionVersion'], + }, + { + name: 'invalid type (boolean)', + partialExecutionVersion: true, + expectedErrorPath: ['partialExecutionVersion'], + }, + { + name: 'invalid type (number)', + partialExecutionVersion: 1, + expectedErrorPath: ['partialExecutionVersion'], + }, + ])('should fail validation for $name', ({ partialExecutionVersion, expectedErrorPath }) => { + const result = ManualRunQueryDto.safeParse({ partialExecutionVersion }); + + if (result.success) { + return fail('expected validation to fail'); + } + + expect(result.error.issues[0].path).toEqual(expectedErrorPath); + }); + }); +}); diff --git a/packages/@n8n/api-types/src/dto/workflows/manual-run-query.dto.ts b/packages/@n8n/api-types/src/dto/workflows/manual-run-query.dto.ts new file mode 100644 index 0000000000..a97ef7d161 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/workflows/manual-run-query.dto.ts @@ -0,0 +1,9 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +export class ManualRunQueryDto extends Z.class({ + partialExecutionVersion: z + .enum(['1', '2']) + .default('1') + .transform((version) => Number.parseInt(version) as 1 | 2), +}) {} diff --git a/packages/@n8n/api-types/src/frontend-settings.ts b/packages/@n8n/api-types/src/frontend-settings.ts index 3ce856d6ad..2a8bb3ea86 100644 --- a/packages/@n8n/api-types/src/frontend-settings.ts +++ b/packages/@n8n/api-types/src/frontend-settings.ts @@ -87,6 +87,7 @@ export interface FrontendSettings { }; }; publicApi: { + apiKeysPerUserLimit: number; enabled: boolean; latestVersion: number; path: string; @@ -177,4 +178,8 @@ export interface FrontendSettings { }; betaFeatures: FrontendBetaFeatures[]; easyAIWorkflowOnboarded: boolean; + partialExecution: { + version: 1 | 2; + enforce: boolean; + }; } diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index a003d54201..a51850cc6c 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -4,9 +4,24 @@ export type * from './push'; export type * from './scaling'; export type * from './frontend-settings'; export type * from './user'; +export type * from './api-keys'; export type { Collaborator } from './push/collaboration'; export type { SendWorkerStatusMessage } from './push/worker'; export type { BannerName } from './schemas/bannerName.schema'; export { passwordSchema } from './schemas/password.schema'; + +export type { + ProjectType, + ProjectIcon, + ProjectRole, + ProjectRelation, +} from './schemas/project.schema'; + +export { + type SourceControlledFile, + SOURCE_CONTROL_FILE_LOCATION, + SOURCE_CONTROL_FILE_STATUS, + SOURCE_CONTROL_FILE_TYPE, +} from './schemas/source-controlled-file.schema'; diff --git a/packages/@n8n/api-types/src/push/execution.ts b/packages/@n8n/api-types/src/push/execution.ts index 320b3dc264..b87bb67d0f 100644 --- a/packages/@n8n/api-types/src/push/execution.ts +++ b/packages/@n8n/api-types/src/push/execution.ts @@ -52,6 +52,17 @@ type NodeExecuteAfter = { executionId: string; nodeName: string; data: ITaskData; + + /** + * When a worker relays updates about a manual execution to main, if the + * payload size is above a limit, we send only a placeholder to the client. + * Later we fetch the entire execution data and fill in any placeholders. + * + * When sending a placheolder, we also send the number of output items, so + * the client knows ahead of time how many items are there, to prevent the + * items count from jumping up when the execution finishes. + */ + itemCount?: number; }; }; diff --git a/packages/@n8n/api-types/src/scaling.ts b/packages/@n8n/api-types/src/scaling.ts index f0c3627e84..30db754a11 100644 --- a/packages/@n8n/api-types/src/scaling.ts +++ b/packages/@n8n/api-types/src/scaling.ts @@ -6,7 +6,7 @@ export type RunningJobSummary = { workflowName: string; mode: WorkflowExecuteMode; startedAt: Date; - retryOf: string; + retryOf?: string; status: ExecutionStatus; }; diff --git a/packages/@n8n/api-types/src/schemas/__tests__/project.schema.test.ts b/packages/@n8n/api-types/src/schemas/__tests__/project.schema.test.ts new file mode 100644 index 0000000000..9a1cf47414 --- /dev/null +++ b/packages/@n8n/api-types/src/schemas/__tests__/project.schema.test.ts @@ -0,0 +1,105 @@ +import { + projectNameSchema, + projectTypeSchema, + projectIconSchema, + projectRoleSchema, + projectRelationSchema, +} from '../project.schema'; + +describe('project.schema', () => { + describe('projectNameSchema', () => { + test.each([ + { name: 'valid name', value: 'My Project', expected: true }, + { name: 'empty name', value: '', expected: false }, + { name: 'name too long', value: 'a'.repeat(256), expected: false }, + ])('should validate $name', ({ value, expected }) => { + const result = projectNameSchema.safeParse(value); + expect(result.success).toBe(expected); + }); + }); + + describe('projectTypeSchema', () => { + test.each([ + { name: 'valid type: personal', value: 'personal', expected: true }, + { name: 'valid type: team', value: 'team', expected: true }, + { name: 'invalid type', value: 'invalid', expected: false }, + ])('should validate $name', ({ value, expected }) => { + const result = projectTypeSchema.safeParse(value); + expect(result.success).toBe(expected); + }); + }); + + describe('projectIconSchema', () => { + test.each([ + { + name: 'valid emoji icon', + value: { type: 'emoji', value: '🚀' }, + expected: true, + }, + { + name: 'valid icon', + value: { type: 'icon', value: 'blah' }, + expected: true, + }, + { + name: 'invalid icon type', + value: { type: 'invalid', value: '🚀' }, + expected: false, + }, + { + name: 'empty icon value', + value: { type: 'emoji', value: '' }, + expected: false, + }, + ])('should validate $name', ({ value, expected }) => { + const result = projectIconSchema.safeParse(value); + expect(result.success).toBe(expected); + }); + }); + + describe('projectRoleSchema', () => { + test.each([ + { name: 'valid role: project:personalOwner', value: 'project:personalOwner', expected: true }, + { name: 'valid role: project:admin', value: 'project:admin', expected: true }, + { name: 'valid role: project:editor', value: 'project:editor', expected: true }, + { name: 'valid role: project:viewer', value: 'project:viewer', expected: true }, + { name: 'invalid role', value: 'invalid-role', expected: false }, + ])('should validate $name', ({ value, expected }) => { + const result = projectRoleSchema.safeParse(value); + expect(result.success).toBe(expected); + }); + }); + + describe('projectRelationSchema', () => { + test.each([ + { + name: 'valid relation', + value: { userId: 'user-123', role: 'project:admin' }, + expected: true, + }, + { + name: 'invalid userId type', + value: { userId: 123, role: 'project:admin' }, + expected: false, + }, + { + name: 'invalid role', + value: { userId: 'user-123', role: 'invalid-role' }, + expected: false, + }, + { + name: 'missing userId', + value: { role: 'project:admin' }, + expected: false, + }, + { + name: 'missing role', + value: { userId: 'user-123' }, + expected: false, + }, + ])('should validate $name', ({ value, expected }) => { + const result = projectRelationSchema.safeParse(value); + expect(result.success).toBe(expected); + }); + }); +}); diff --git a/packages/@n8n/api-types/src/schemas/project.schema.ts b/packages/@n8n/api-types/src/schemas/project.schema.ts new file mode 100644 index 0000000000..11c6cc2b37 --- /dev/null +++ b/packages/@n8n/api-types/src/schemas/project.schema.ts @@ -0,0 +1,26 @@ +import { z } from 'zod'; + +export const projectNameSchema = z.string().min(1).max(255); + +export const projectTypeSchema = z.enum(['personal', 'team']); +export type ProjectType = z.infer; + +export const projectIconSchema = z.object({ + type: z.enum(['emoji', 'icon']), + value: z.string().min(1), +}); +export type ProjectIcon = z.infer; + +export const projectRoleSchema = z.enum([ + 'project:personalOwner', // personalOwner is only used for personal projects + 'project:admin', + 'project:editor', + 'project:viewer', +]); +export type ProjectRole = z.infer; + +export const projectRelationSchema = z.object({ + userId: z.string(), + role: projectRoleSchema, +}); +export type ProjectRelation = z.infer; diff --git a/packages/@n8n/api-types/src/schemas/source-controlled-file.schema.ts b/packages/@n8n/api-types/src/schemas/source-controlled-file.schema.ts new file mode 100644 index 0000000000..31f2db8f92 --- /dev/null +++ b/packages/@n8n/api-types/src/schemas/source-controlled-file.schema.ts @@ -0,0 +1,34 @@ +import { z } from 'zod'; + +const FileTypeSchema = z.enum(['credential', 'workflow', 'tags', 'variables', 'file']); +export const SOURCE_CONTROL_FILE_TYPE = FileTypeSchema.Values; + +const FileStatusSchema = z.enum([ + 'new', + 'modified', + 'deleted', + 'created', + 'renamed', + 'conflicted', + 'ignored', + 'staged', + 'unknown', +]); +export const SOURCE_CONTROL_FILE_STATUS = FileStatusSchema.Values; + +const FileLocationSchema = z.enum(['local', 'remote']); +export const SOURCE_CONTROL_FILE_LOCATION = FileLocationSchema.Values; + +export const SourceControlledFileSchema = z.object({ + file: z.string(), + id: z.string(), + name: z.string(), + type: FileTypeSchema, + status: FileStatusSchema, + location: FileLocationSchema, + conflict: z.boolean(), + updatedAt: z.string(), + pushed: z.boolean().optional(), +}); + +export type SourceControlledFile = z.infer; diff --git a/packages/@n8n/benchmark/Dockerfile b/packages/@n8n/benchmark/Dockerfile index 9525a9a4c2..8275bbe415 100644 --- a/packages/@n8n/benchmark/Dockerfile +++ b/packages/@n8n/benchmark/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && \ ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" -RUN corepack enable +RUN npm install -g corepack@0.31 && corepack enable # # Builder diff --git a/packages/@n8n/benchmark/README.md b/packages/@n8n/benchmark/README.md index af8726d0ea..70c20f9731 100644 --- a/packages/@n8n/benchmark/README.md +++ b/packages/@n8n/benchmark/README.md @@ -27,7 +27,7 @@ docker run ghcr.io/n8n-io/n8n-benchmark:latest run \ --n8nUserPassword=InstanceOwnerPassword \ --vus=5 \ --duration=1m \ - --scenarioFilter SingleWebhook + --scenarioFilter=single-webhook ``` ### Using custom scenarios with the Docker image diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json index f42bd7c508..c67ed3ec35 100644 --- a/packages/@n8n/benchmark/package.json +++ b/packages/@n8n/benchmark/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/n8n-benchmark", - "version": "1.9.0", + "version": "1.11.0", "description": "Cli for running benchmark tests for n8n", "main": "dist/index", "scripts": { diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/postgres/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8n-setups/postgres/docker-compose.yml index 3cc08227c1..5241971325 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/postgres/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8n-setups/postgres/docker-compose.yml @@ -32,6 +32,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal ports: - 5678:5678 volumes: diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml index c686f581b3..723c10f00e 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-multi-main/docker-compose.yml @@ -50,6 +50,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal command: worker volumes: - ${RUN_DIR}/n8n-worker1:/n8n @@ -82,6 +85,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal command: worker volumes: - ${RUN_DIR}/n8n-worker2:/n8n @@ -117,6 +123,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal volumes: - ${RUN_DIR}/n8n-main2:/n8n depends_on: @@ -154,7 +163,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password - + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal volumes: - ${RUN_DIR}/n8n-main1:/n8n depends_on: diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-single-main/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-single-main/docker-compose.yml index fe9e3a26c0..ecefddf5b0 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/scaling-single-main/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8n-setups/scaling-single-main/docker-compose.yml @@ -48,6 +48,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal command: worker volumes: - ${RUN_DIR}/n8n-worker1:/n8n @@ -78,6 +81,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal command: worker volumes: - ${RUN_DIR}/n8n-worker2:/n8n @@ -109,6 +115,9 @@ services: - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres - DB_POSTGRESDB_PASSWORD=password + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal ports: - 5678:5678 volumes: diff --git a/packages/@n8n/benchmark/scripts/n8n-setups/sqlite/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8n-setups/sqlite/docker-compose.yml index 20ec0067fa..37e1424cfd 100644 --- a/packages/@n8n/benchmark/scripts/n8n-setups/sqlite/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8n-setups/sqlite/docker-compose.yml @@ -14,6 +14,9 @@ services: - N8N_USER_FOLDER=/n8n - DB_SQLITE_POOL_SIZE=3 - DB_SQLITE_ENABLE_WAL=true + # Task Runner config + - N8N_RUNNERS_ENABLED=true + - N8N_RUNNERS_MODE=internal ports: - 5678:5678 volumes: diff --git a/packages/@n8n/chat/README.md b/packages/@n8n/chat/README.md index 6299c80d12..4501b86cc5 100644 --- a/packages/@n8n/chat/README.md +++ b/packages/@n8n/chat/README.md @@ -112,6 +112,7 @@ createChat({ mode: 'window', chatInputKey: 'chatInput', chatSessionKey: 'sessionId', + loadPreviousSession: true, metadata: {}, showWelcomeScreen: false, defaultLanguage: 'en', @@ -161,15 +162,20 @@ createChat({ - **Default**: `false` - **Description**: Whether to show the welcome screen when the Chat window is opened. +### `chatInputKey` +- **Type**: `string` +- **Default**: `'chatInput'` +- **Description**: The key to use for sending the chat input for the AI Agent node. + ### `chatSessionKey` - **Type**: `string` - **Default**: `'sessionId'` - **Description**: The key to use for sending the chat history session ID for the AI Memory node. -### `chatInputKey` -- **Type**: `string` -- **Default**: `'chatInput'` -- **Description**: The key to use for sending the chat input for the AI Agent node. +### `loadPreviousSession` +- **Type**: `boolean` +- **Default**: `true` +- **Description**: Whether to load previous messages (chat context). ### `defaultLanguage` - **Type**: `string` diff --git a/packages/@n8n/client-oauth2/.eslintrc.js b/packages/@n8n/client-oauth2/.eslintrc.js index c3fe283453..be8ebd21d1 100644 --- a/packages/@n8n/client-oauth2/.eslintrc.js +++ b/packages/@n8n/client-oauth2/.eslintrc.js @@ -11,5 +11,6 @@ module.exports = { rules: { '@typescript-eslint/consistent-type-imports': 'error', 'n8n-local-rules/no-plain-errors': 'off', + 'n8n-local-rules/no-uncaught-json-parse': 'off', }, }; diff --git a/packages/@n8n/client-oauth2/package.json b/packages/@n8n/client-oauth2/package.json index 33f15218f2..fccbc63098 100644 --- a/packages/@n8n/client-oauth2/package.json +++ b/packages/@n8n/client-oauth2/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/client-oauth2", - "version": "0.21.0", + "version": "0.22.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/client-oauth2/src/ClientOAuth2.ts b/packages/@n8n/client-oauth2/src/ClientOAuth2.ts index 676249254a..62e0241e6f 100644 --- a/packages/@n8n/client-oauth2/src/ClientOAuth2.ts +++ b/packages/@n8n/client-oauth2/src/ClientOAuth2.ts @@ -1,8 +1,6 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-argument */ /* eslint-disable @typescript-eslint/no-explicit-any */ import axios from 'axios'; -import type { AxiosRequestConfig } from 'axios'; +import type { AxiosRequestConfig, AxiosResponse } from 'axios'; import { Agent } from 'https'; import * as qs from 'querystring'; @@ -10,7 +8,7 @@ import type { ClientOAuth2TokenData } from './ClientOAuth2Token'; import { ClientOAuth2Token } from './ClientOAuth2Token'; import { CodeFlow } from './CodeFlow'; import { CredentialsFlow } from './CredentialsFlow'; -import type { Headers } from './types'; +import type { Headers, OAuth2AccessTokenErrorResponse } from './types'; import { getAuthError } from './utils'; export interface ClientOAuth2RequestObject { @@ -38,10 +36,10 @@ export interface ClientOAuth2Options { ignoreSSLIssues?: boolean; } -class ResponseError extends Error { +export class ResponseError extends Error { constructor( readonly status: number, - readonly body: object, + readonly body: unknown, readonly code = 'ESTATUS', ) { super(`HTTP status ${status}`); @@ -74,21 +72,12 @@ export class ClientOAuth2 { } /** - * Attempt to parse response body as JSON, fall back to parsing as a query string. + * Request an access token from the OAuth2 server. + * + * @throws {ResponseError} If the response is an unexpected status code. + * @throws {AuthError} If the response is an authentication error. */ - private parseResponseBody(body: string): T { - try { - return JSON.parse(body); - } catch (e) { - return qs.parse(body) as T; - } - } - - /** - * Using the built-in request method, we'll automatically attempt to parse - * the response. - */ - async request(options: ClientOAuth2RequestObject): Promise { + async accessTokenRequest(options: ClientOAuth2RequestObject): Promise { let url = options.url; const query = qs.stringify(options.query); @@ -101,7 +90,7 @@ export class ClientOAuth2 { method: options.method, data: qs.stringify(options.body), headers: options.headers, - transformResponse: (res) => res, + transformResponse: (res: unknown) => res, // Axios rejects the promise by default for all status codes 4xx. // We override this to reject promises only on 5xxs validateStatus: (status) => status < 500, @@ -113,16 +102,36 @@ export class ClientOAuth2 { const response = await axios.request(requestConfig); - const body = this.parseResponseBody(response.data); + if (response.status >= 400) { + const body = this.parseResponseBody(response); + const authErr = getAuthError(body); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const authErr = getAuthError(body); - if (authErr) throw authErr; + if (authErr) throw authErr; + else throw new ResponseError(response.status, response.data); + } - if (response.status < 200 || response.status >= 399) + if (response.status >= 300) { throw new ResponseError(response.status, response.data); + } - return body; + return this.parseResponseBody(response); + } + + /** + * Attempt to parse response body based on the content type. + */ + private parseResponseBody(response: AxiosResponse): T { + const contentType = (response.headers['content-type'] as string) ?? ''; + const body = response.data as string; + + if (contentType.startsWith('application/json')) { + return JSON.parse(body) as T; + } + + if (contentType.startsWith('application/x-www-form-urlencoded')) { + return qs.parse(body) as T; + } + + throw new Error(`Unsupported content type: ${contentType}`); } } diff --git a/packages/@n8n/client-oauth2/src/ClientOAuth2Token.ts b/packages/@n8n/client-oauth2/src/ClientOAuth2Token.ts index 2bcacdf112..d7a80a1699 100644 --- a/packages/@n8n/client-oauth2/src/ClientOAuth2Token.ts +++ b/packages/@n8n/client-oauth2/src/ClientOAuth2Token.ts @@ -1,3 +1,5 @@ +import * as a from 'node:assert'; + import type { ClientOAuth2, ClientOAuth2Options, ClientOAuth2RequestObject } from './ClientOAuth2'; import { DEFAULT_HEADERS } from './constants'; import { auth, expects, getRequestOptions } from './utils'; @@ -65,17 +67,16 @@ export class ClientOAuth2Token { } /** - * Refresh a user access token with the supplied token. + * Refresh a user access token with the refresh token. + * As in RFC 6749 Section 6: https://www.rfc-editor.org/rfc/rfc6749.html#section-6 */ async refresh(opts?: ClientOAuth2Options): Promise { const options = { ...this.client.options, ...opts }; expects(options, 'clientSecret'); + a.ok(this.refreshToken, 'refreshToken is required'); - if (!this.refreshToken) throw new Error('No refresh token'); - - const clientId = options.clientId; - const clientSecret = options.clientSecret; + const { clientId, clientSecret } = options; const headers = { ...DEFAULT_HEADERS }; const body: Record = { refresh_token: this.refreshToken, @@ -99,7 +100,7 @@ export class ClientOAuth2Token { options, ); - const responseData = await this.client.request(requestOptions); + const responseData = await this.client.accessTokenRequest(requestOptions); return this.client.createToken({ ...this.data, ...responseData }); } diff --git a/packages/@n8n/client-oauth2/src/CodeFlow.ts b/packages/@n8n/client-oauth2/src/CodeFlow.ts index 6d0fff235e..6db98929f9 100644 --- a/packages/@n8n/client-oauth2/src/CodeFlow.ts +++ b/packages/@n8n/client-oauth2/src/CodeFlow.ts @@ -1,7 +1,7 @@ import * as qs from 'querystring'; import type { ClientOAuth2, ClientOAuth2Options } from './ClientOAuth2'; -import type { ClientOAuth2Token, ClientOAuth2TokenData } from './ClientOAuth2Token'; +import type { ClientOAuth2Token } from './ClientOAuth2Token'; import { DEFAULT_HEADERS, DEFAULT_URL_BASE } from './constants'; import { auth, expects, getAuthError, getRequestOptions } from './utils'; @@ -117,7 +117,7 @@ export class CodeFlow { options, ); - const responseData = await this.client.request(requestOptions); + const responseData = await this.client.accessTokenRequest(requestOptions); return this.client.createToken(responseData); } } diff --git a/packages/@n8n/client-oauth2/src/CredentialsFlow.ts b/packages/@n8n/client-oauth2/src/CredentialsFlow.ts index f1ccc256e7..eeb5550cf3 100644 --- a/packages/@n8n/client-oauth2/src/CredentialsFlow.ts +++ b/packages/@n8n/client-oauth2/src/CredentialsFlow.ts @@ -1,5 +1,5 @@ import type { ClientOAuth2 } from './ClientOAuth2'; -import type { ClientOAuth2Token, ClientOAuth2TokenData } from './ClientOAuth2Token'; +import type { ClientOAuth2Token } from './ClientOAuth2Token'; import { DEFAULT_HEADERS } from './constants'; import type { Headers } from './types'; import { auth, expects, getRequestOptions } from './utils'; @@ -55,7 +55,7 @@ export class CredentialsFlow { options, ); - const responseData = await this.client.request(requestOptions); + const responseData = await this.client.accessTokenRequest(requestOptions); return this.client.createToken(responseData); } } diff --git a/packages/@n8n/client-oauth2/src/types.ts b/packages/@n8n/client-oauth2/src/types.ts index 69c225d827..26a90bd441 100644 --- a/packages/@n8n/client-oauth2/src/types.ts +++ b/packages/@n8n/client-oauth2/src/types.ts @@ -17,3 +17,14 @@ export interface OAuth2CredentialData { refresh_token?: string; }; } + +/** + * The response from the OAuth2 server when the access token is not successfully + * retrieved. As specified in RFC 6749 Section 5.2: + * https://www.rfc-editor.org/rfc/rfc6749.html#section-5.2 + */ +export interface OAuth2AccessTokenErrorResponse extends Record { + error: string; + error_description?: string; + error_uri?: string; +} diff --git a/packages/@n8n/client-oauth2/test/ClientOAuth2.test.ts b/packages/@n8n/client-oauth2/test/ClientOAuth2.test.ts new file mode 100644 index 0000000000..7e6fa788be --- /dev/null +++ b/packages/@n8n/client-oauth2/test/ClientOAuth2.test.ts @@ -0,0 +1,168 @@ +import axios from 'axios'; +import nock from 'nock'; + +import { ClientOAuth2, ResponseError } from '@/ClientOAuth2'; +import { ERROR_RESPONSES } from '@/constants'; +import { auth, AuthError } from '@/utils'; + +import * as config from './config'; + +describe('ClientOAuth2', () => { + const client = new ClientOAuth2({ + clientId: config.clientId, + clientSecret: config.clientSecret, + accessTokenUri: config.accessTokenUri, + authentication: 'header', + }); + + beforeAll(async () => { + nock.disableNetConnect(); + }); + + afterAll(() => { + nock.restore(); + }); + + describe('accessTokenRequest', () => { + const authHeader = auth(config.clientId, config.clientSecret); + + const makeTokenCall = async () => + await client.accessTokenRequest({ + url: config.accessTokenUri, + method: 'POST', + headers: { + Authorization: authHeader, + Accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: { + refresh_token: 'test', + grant_type: 'refresh_token', + }, + }); + + const mockTokenResponse = ({ + status = 200, + headers, + body, + }: { + status: number; + body: string; + headers: Record; + }) => + nock(config.baseUrl).post('/login/oauth/access_token').once().reply(status, body, headers); + + it('should send the correct request based on given options', async () => { + mockTokenResponse({ + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + access_token: config.accessToken, + refresh_token: config.refreshToken, + }), + }); + + const axiosSpy = jest.spyOn(axios, 'request'); + + await makeTokenCall(); + + expect(axiosSpy).toHaveBeenCalledWith( + expect.objectContaining({ + url: config.accessTokenUri, + method: 'POST', + data: 'refresh_token=test&grant_type=refresh_token', + headers: { + Authorization: authHeader, + Accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }), + ); + }); + + test.each([ + { + contentType: 'application/json', + body: JSON.stringify({ + access_token: config.accessToken, + refresh_token: config.refreshToken, + }), + }, + { + contentType: 'application/json; charset=utf-8', + body: JSON.stringify({ + access_token: config.accessToken, + refresh_token: config.refreshToken, + }), + }, + { + contentType: 'application/x-www-form-urlencoded', + body: `access_token=${config.accessToken}&refresh_token=${config.refreshToken}`, + }, + ])('should parse response with content type $contentType', async ({ contentType, body }) => { + mockTokenResponse({ + status: 200, + headers: { 'Content-Type': contentType }, + body, + }); + + const response = await makeTokenCall(); + + expect(response).toEqual({ + access_token: config.accessToken, + refresh_token: config.refreshToken, + }); + }); + + test.each([ + { + contentType: 'text/html', + body: 'Hello, world!', + }, + { + contentType: 'application/xml', + body: 'Hello, world!', + }, + { + contentType: 'text/plain', + body: 'Hello, world!', + }, + ])('should reject content type $contentType', async ({ contentType, body }) => { + mockTokenResponse({ + status: 200, + headers: { 'Content-Type': contentType }, + body, + }); + + const result = await makeTokenCall().catch((err) => err); + expect(result).toBeInstanceOf(Error); + expect(result.message).toEqual(`Unsupported content type: ${contentType}`); + }); + + it('should reject 4xx responses with auth errors', async () => { + mockTokenResponse({ + status: 401, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ error: 'access_denied' }), + }); + + const result = await makeTokenCall().catch((err) => err); + expect(result).toBeInstanceOf(AuthError); + expect(result.message).toEqual(ERROR_RESPONSES.access_denied); + expect(result.body).toEqual({ error: 'access_denied' }); + }); + + it('should reject 3xx responses with response errors', async () => { + mockTokenResponse({ + status: 302, + headers: {}, + body: 'Redirected', + }); + + const result = await makeTokenCall().catch((err) => err); + expect(result).toBeInstanceOf(ResponseError); + expect(result.message).toEqual('HTTP status 302'); + expect(result.body).toEqual('Redirected'); + }); + }); +}); diff --git a/packages/@n8n/codemirror-lang/src/expressions/expressions.grammar b/packages/@n8n/codemirror-lang/src/expressions/expressions.grammar index 9217f2c2fb..6e9efee787 100644 --- a/packages/@n8n/codemirror-lang/src/expressions/expressions.grammar +++ b/packages/@n8n/codemirror-lang/src/expressions/expressions.grammar @@ -15,7 +15,7 @@ entity { Plaintext | Resolvable } resolvableChar { unicodeChar | "}" ![}] | "\\}}" } - unicodeChar { $[\u0000-\u007C] | $[\u007E-\u20CF] | $[\u{1F300}-\u{1F64F}] | $[\u4E00-\u9FFF] } + unicodeChar { $[\u0000-\u007C] | $[\u007E-\u20CF] | $[\u{1F300}-\u{1FAF8}] | $[\u4E00-\u9FFF] } } @detectDelim diff --git a/packages/@n8n/codemirror-lang/src/expressions/grammar.ts b/packages/@n8n/codemirror-lang/src/expressions/grammar.ts index bd081b4832..fc3a2c9e31 100644 --- a/packages/@n8n/codemirror-lang/src/expressions/grammar.ts +++ b/packages/@n8n/codemirror-lang/src/expressions/grammar.ts @@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({ skippedNodes: [0], repeatNodeCount: 1, tokenData: - "&U~RTO#ob#o#p!h#p;'Sb;'S;=`!]<%lOb~gTQ~O#ob#o#pv#p;'Sb;'S;=`!]<%lOb~yUO#ob#p;'Sb;'S;=`!]<%l~b~Ob~~!c~!`P;=`<%lb~!hOQ~~!kVO#ob#o#p#Q#p;'Sb;'S;=`!]<%l~b~Ob~~!c~#TWO#O#Q#O#P#m#P#q#Q#q#r%Z#r$Ml#Q*5S41d#Q;(b;(c%x;(c;(d&O~#pWO#O#Q#O#P#m#P#q#Q#q#r$Y#r$Ml#Q*5S41d#Q;(b;(c%x;(c;(d&O~$]TO#q#Q#q#r$l#r;'S#Q;'S;=`%r<%lO#Q~$qWR~O#O#Q#O#P#m#P#q#Q#q#r%Z#r$Ml#Q*5S41d#Q;(b;(c%x;(c;(d&O~%^TO#q#Q#q#r%m#r;'S#Q;'S;=`%r<%lO#Q~%rOR~~%uP;=`<%l#Q~%{P;NQ<%l#Q~&RP;=`;JY#Q", + "&_~RTO#ob#o#p!h#p;'Sb;'S;=`!]<%lOb~gTQ~O#ob#o#pv#p;'Sb;'S;=`!]<%lOb~yUO#ob#p;'Sb;'S;=`!]<%l~b~Ob~~!c~!`P;=`<%lb~!hOQ~~!kVO#ob#o#p#Q#p;'Sb;'S;=`!]<%l~b~Ob~~!c~#TXO#O#Q#O#P#p#P#q#Q#q#r%d#r$Ml#Q*5S41d#Q;(b;(c&R;(c;(d%{;(d;(e&X~#sXO#O#Q#O#P#p#P#q#Q#q#r$`#r$Ml#Q*5S41d#Q;(b;(c&R;(c;(d%{;(d;(e&X~$cTO#q#Q#q#r$r#r;'S#Q;'S;=`%{<%lO#Q~$wXR~O#O#Q#O#P#p#P#q#Q#q#r%d#r$Ml#Q*5S41d#Q;(b;(c&R;(c;(d%{;(d;(e&X~%gTO#q#Q#q#r%v#r;'S#Q;'S;=`%{<%lO#Q~%{OR~~&OP;=`<%l#Q~&UP;NQ<%l#Q~&[P;=`;My#Q", tokenizers: [0], topRules: { Program: [0, 1] }, tokenPrec: 0, diff --git a/packages/@n8n/codemirror-lang/test/expressions/cases.txt b/packages/@n8n/codemirror-lang/test/expressions/cases.txt index 36f41ddccd..37db1e0bc0 100644 --- a/packages/@n8n/codemirror-lang/test/expressions/cases.txt +++ b/packages/@n8n/codemirror-lang/test/expressions/cases.txt @@ -277,3 +277,19 @@ Program(Resolvable) ==> Program(Resolvable) + +# Resolvable with new emoji range + +{{ '🟢' }} + +==> + +Program(Resolvable) + +# Resolvable with new emoji range end of range + +{{ '🫸' }} + +==> + +Program(Resolvable) diff --git a/packages/@n8n/config/package.json b/packages/@n8n/config/package.json index f32da81581..0c58861309 100644 --- a/packages/@n8n/config/package.json +++ b/packages/@n8n/config/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/config", - "version": "1.24.0", + "version": "1.28.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/config/src/configs/database.config.ts b/packages/@n8n/config/src/configs/database.config.ts index cebbf2191a..dc8bdde98d 100644 --- a/packages/@n8n/config/src/configs/database.config.ts +++ b/packages/@n8n/config/src/configs/database.config.ts @@ -107,7 +107,7 @@ class MysqlConfig { } @Config -class SqliteConfig { +export class SqliteConfig { /** SQLite database file name */ @Env('DB_SQLITE_DATABASE') database: string = 'database.sqlite'; diff --git a/packages/@n8n/config/src/configs/external-hooks.config.ts b/packages/@n8n/config/src/configs/external-hooks.config.ts new file mode 100644 index 0000000000..20c8eb8654 --- /dev/null +++ b/packages/@n8n/config/src/configs/external-hooks.config.ts @@ -0,0 +1,17 @@ +import { Config, Env } from '../decorators'; + +class ColonSeparatedStringArray extends Array { + constructor(str: string) { + super(); + const parsed = str.split(':') as this; + const filtered = parsed.filter((i) => typeof i === 'string' && i.length); + return filtered.length ? filtered : []; + } +} + +@Config +export class ExternalHooksConfig { + /** Files containing external hooks. Multiple files can be separated by colon (":") */ + @Env('EXTERNAL_HOOK_FILES') + files: ColonSeparatedStringArray = []; +} diff --git a/packages/@n8n/config/src/configs/external-storage.config.ts b/packages/@n8n/config/src/configs/external-storage.config.ts index 6e5fbd64d8..aff2447d40 100644 --- a/packages/@n8n/config/src/configs/external-storage.config.ts +++ b/packages/@n8n/config/src/configs/external-storage.config.ts @@ -23,11 +23,14 @@ class S3CredentialsConfig { } @Config -class S3Config { +export class S3Config { /** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */ @Env('N8N_EXTERNAL_STORAGE_S3_HOST') host: string = ''; + @Env('N8N_EXTERNAL_STORAGE_S3_PROTOCOL') + protocol: 'http' | 'https' = 'https'; + @Nested bucket: S3BucketConfig; diff --git a/packages/@n8n/config/src/configs/nodes.config.ts b/packages/@n8n/config/src/configs/nodes.config.ts index 577c4055ab..f5e190e289 100644 --- a/packages/@n8n/config/src/configs/nodes.config.ts +++ b/packages/@n8n/config/src/configs/nodes.config.ts @@ -33,6 +33,10 @@ class CommunityPackagesConfig { /** Whether to reinstall any missing community packages */ @Env('N8N_REINSTALL_MISSING_PACKAGES') reinstallMissing: boolean = false; + + /** Whether to allow community packages as tools for AI agents */ + @Env('N8N_COMMUNITY_PACKAGES_ALLOW_TOOL_USAGE') + allowToolUsage: boolean = false; } @Config diff --git a/packages/@n8n/config/src/configs/partial-executions.config.ts b/packages/@n8n/config/src/configs/partial-executions.config.ts new file mode 100644 index 0000000000..7937f451d3 --- /dev/null +++ b/packages/@n8n/config/src/configs/partial-executions.config.ts @@ -0,0 +1,12 @@ +import { Config, Env } from '../decorators'; + +@Config +export class PartialExecutionsConfig { + /** Partial execution logic version to use by default. */ + @Env('N8N_PARTIAL_EXECUTION_VERSION_DEFAULT') + version: 1 | 2 = 1; + + /** Set this to true to enforce using the default version. Users cannot use the other version then by setting a local storage key. */ + @Env('N8N_PARTIAL_EXECUTION_ENFORCE_VERSION') + enforce: boolean = false; +} diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts index 02ebdf5df9..af7e911877 100644 --- a/packages/@n8n/config/src/configs/runners.config.ts +++ b/packages/@n8n/config/src/configs/runners.config.ts @@ -12,7 +12,6 @@ export class TaskRunnersConfig { @Env('N8N_RUNNERS_ENABLED') enabled: boolean = false; - // Defaults to true for now @Env('N8N_RUNNERS_MODE') mode: TaskRunnerMode = 'internal'; @@ -23,12 +22,12 @@ export class TaskRunnersConfig { @Env('N8N_RUNNERS_AUTH_TOKEN') authToken: string = ''; - /** IP address task runners server should listen on */ - @Env('N8N_RUNNERS_HEALTH_CHECK_SERVER_PORT') + /** IP address task runners broker should listen on */ + @Env('N8N_RUNNERS_BROKER_PORT') port: number = 5679; - /** IP address task runners server should listen on */ - @Env('N8N_RUNNERS_SERVER_LISTEN_ADDRESS') + /** IP address task runners broker should listen on */ + @Env('N8N_RUNNERS_BROKER_LISTEN_ADDRESS') listenAddress: string = '127.0.0.1'; /** Maximum size of a payload sent to the runner in bytes, Default 1G */ diff --git a/packages/@n8n/config/src/configs/scaling-mode.config.ts b/packages/@n8n/config/src/configs/scaling-mode.config.ts index f202440a5b..e4331ce9d0 100644 --- a/packages/@n8n/config/src/configs/scaling-mode.config.ts +++ b/packages/@n8n/config/src/configs/scaling-mode.config.ts @@ -52,6 +52,10 @@ class RedisConfig { /** Whether to enable TLS on Redis connections. */ @Env('QUEUE_BULL_REDIS_TLS') tls: boolean = false; + + /** Whether to enable dual-stack hostname resolution for Redis connections. */ + @Env('QUEUE_BULL_REDIS_DUALSTACK') + dualStack: boolean = false; } @Config diff --git a/packages/@n8n/config/src/configs/sentry.config.ts b/packages/@n8n/config/src/configs/sentry.config.ts index d1067f9984..97e34edeea 100644 --- a/packages/@n8n/config/src/configs/sentry.config.ts +++ b/packages/@n8n/config/src/configs/sentry.config.ts @@ -2,11 +2,35 @@ import { Config, Env } from '../decorators'; @Config export class SentryConfig { - /** Sentry DSN for the backend. */ + /** Sentry DSN (data source name) for the backend. */ @Env('N8N_SENTRY_DSN') backendDsn: string = ''; - /** Sentry DSN for the frontend . */ + /** Sentry DSN (data source name) for the frontend. */ @Env('N8N_FRONTEND_SENTRY_DSN') frontendDsn: string = ''; + + /** + * Version of the n8n instance + * + * @example '1.73.0' + */ + @Env('N8N_VERSION') + n8nVersion: string = ''; + + /** + * Environment of the n8n instance. + * + * @example 'production' + */ + @Env('ENVIRONMENT') + environment: string = ''; + + /** + * Name of the deployment, e.g. cloud account name. + * + * @example 'janober' + */ + @Env('DEPLOYMENT_NAME') + deploymentName: string = ''; } diff --git a/packages/@n8n/config/src/configs/tags.config.ts b/packages/@n8n/config/src/configs/tags.config.ts new file mode 100644 index 0000000000..1c6db71809 --- /dev/null +++ b/packages/@n8n/config/src/configs/tags.config.ts @@ -0,0 +1,10 @@ +import { Config, Env } from '../decorators'; + +@Config +export class TagsConfig { + /* + Disable workflow tags + */ + @Env('N8N_WORKFLOW_TAGS_DISABLED') + disabled: boolean = false; +} diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index 945b5f1237..edcc794ca5 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -6,6 +6,7 @@ import { DiagnosticsConfig } from './configs/diagnostics.config'; import { EndpointsConfig } from './configs/endpoints.config'; import { EventBusConfig } from './configs/event-bus.config'; import { ExecutionsConfig } from './configs/executions.config'; +import { ExternalHooksConfig } from './configs/external-hooks.config'; import { ExternalSecretsConfig } from './configs/external-secrets.config'; import { ExternalStorageConfig } from './configs/external-storage.config'; import { GenericConfig } from './configs/generic.config'; @@ -13,11 +14,13 @@ import { LicenseConfig } from './configs/license.config'; import { LoggingConfig } from './configs/logging.config'; import { MultiMainSetupConfig } from './configs/multi-main-setup.config'; import { NodesConfig } from './configs/nodes.config'; +import { PartialExecutionsConfig } from './configs/partial-executions.config'; import { PublicApiConfig } from './configs/public-api.config'; import { TaskRunnersConfig } from './configs/runners.config'; import { ScalingModeConfig } from './configs/scaling-mode.config'; import { SecurityConfig } from './configs/security.config'; import { SentryConfig } from './configs/sentry.config'; +import { TagsConfig } from './configs/tags.config'; import { TemplatesConfig } from './configs/templates.config'; import { UserManagementConfig } from './configs/user-management.config'; import { VersionNotificationsConfig } from './configs/version-notifications.config'; @@ -29,6 +32,7 @@ export { TaskRunnersConfig } from './configs/runners.config'; export { SecurityConfig } from './configs/security.config'; export { ExecutionsConfig } from './configs/executions.config'; export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config'; +export { S3Config } from './configs/external-storage.config'; export { LOG_SCOPES } from './configs/logging.config'; export type { LogScope } from './configs/logging.config'; @@ -49,6 +53,9 @@ export class GlobalConfig { @Nested publicApi: PublicApiConfig; + @Nested + externalHooks: ExternalHooksConfig; + @Nested externalSecrets: ExternalSecretsConfig; @@ -125,4 +132,10 @@ export class GlobalConfig { @Nested aiAssistant: AiAssistantConfig; + + @Nested + tags: TagsConfig; + + @Nested + partialExecutions: PartialExecutionsConfig; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index ed386fee6b..32952ec60b 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -107,6 +107,9 @@ describe('GlobalConfig', () => { maxFileSizeInKB: 10240, }, }, + externalHooks: { + files: [], + }, externalSecrets: { preferGet: false, updateInterval: 300, @@ -116,6 +119,7 @@ describe('GlobalConfig', () => { enabled: true, registry: 'https://registry.npmjs.org', reinstallMissing: false, + allowToolUsage: false, }, errorTriggerType: 'n8n-nodes-base.errorTrigger', include: [], @@ -138,6 +142,7 @@ describe('GlobalConfig', () => { externalStorage: { s3: { host: '', + protocol: 'https', bucket: { name: '', region: '', @@ -209,6 +214,7 @@ describe('GlobalConfig', () => { username: '', clusterNodes: '', tls: false, + dualStack: false, }, gracefulShutdownTimeout: 30, prefix: 'bull', @@ -236,6 +242,9 @@ describe('GlobalConfig', () => { sentry: { backendDsn: '', frontendDsn: '', + n8nVersion: '', + environment: '', + deploymentName: '', }, logging: { level: 'info', @@ -292,6 +301,13 @@ describe('GlobalConfig', () => { aiAssistant: { baseUrl: '', }, + tags: { + disabled: false, + }, + partialExecutions: { + version: 1, + enforce: false, + }, }; it('should use all default values when no env variables are defined', () => { diff --git a/packages/@n8n/di/jest.config.js b/packages/@n8n/di/jest.config.js index d6c48554a7..d14f2d60c6 100644 --- a/packages/@n8n/di/jest.config.js +++ b/packages/@n8n/di/jest.config.js @@ -1,2 +1,7 @@ /** @type {import('jest').Config} */ -module.exports = require('../../../jest.config'); +module.exports = { + ...require('../../../jest.config'), + transform: { + '^.+\\.ts$': ['ts-jest', { isolatedModules: false }], + }, +}; diff --git a/packages/@n8n/di/package.json b/packages/@n8n/di/package.json index be327aefb6..88774017d3 100644 --- a/packages/@n8n/di/package.json +++ b/packages/@n8n/di/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/di", - "version": "0.2.0", + "version": "0.3.0", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", diff --git a/packages/@n8n/di/src/__tests__/circular-depedency.test.ts b/packages/@n8n/di/src/__tests__/circular-depedency.test.ts new file mode 100644 index 0000000000..66bce38f7b --- /dev/null +++ b/packages/@n8n/di/src/__tests__/circular-depedency.test.ts @@ -0,0 +1,17 @@ +import { ServiceA } from './fixtures/ServiceA'; +import { ServiceB } from './fixtures/ServiceB'; +import { Container } from '../di'; + +describe('DI Container', () => { + describe('circular dependency', () => { + it('should detect multilevel circular dependencies', () => { + expect(() => Container.get(ServiceA)).toThrow( + '[DI] Circular dependency detected in ServiceB at index 0.\nServiceA -> ServiceB', + ); + + expect(() => Container.get(ServiceB)).toThrow( + '[DI] Circular dependency detected in ServiceB at index 0.\nServiceB', + ); + }); + }); +}); diff --git a/packages/@n8n/di/src/__tests__/fixtures/ServiceA.ts b/packages/@n8n/di/src/__tests__/fixtures/ServiceA.ts new file mode 100644 index 0000000000..83f4c90430 --- /dev/null +++ b/packages/@n8n/di/src/__tests__/fixtures/ServiceA.ts @@ -0,0 +1,8 @@ +// eslint-disable-next-line import/no-cycle +import { ServiceB } from './ServiceB'; +import { Service } from '../../di'; + +@Service() +export class ServiceA { + constructor(readonly b: ServiceB) {} +} diff --git a/packages/@n8n/di/src/__tests__/fixtures/ServiceB.ts b/packages/@n8n/di/src/__tests__/fixtures/ServiceB.ts new file mode 100644 index 0000000000..a0dbd1908a --- /dev/null +++ b/packages/@n8n/di/src/__tests__/fixtures/ServiceB.ts @@ -0,0 +1,8 @@ +// eslint-disable-next-line import/no-cycle +import { ServiceA } from './ServiceA'; +import { Service } from '../../di'; + +@Service() +export class ServiceB { + constructor(readonly a: ServiceA) {} +} diff --git a/packages/@n8n/di/src/di.ts b/packages/@n8n/di/src/di.ts index a4acb98474..08d86eae07 100644 --- a/packages/@n8n/di/src/di.ts +++ b/packages/@n8n/di/src/di.ts @@ -78,13 +78,6 @@ class ContainerClass { if (metadata?.instance) return metadata.instance as T; - // Check for circular dependencies before proceeding with instantiation - if (resolutionStack.includes(type)) { - throw new DIError( - `Circular dependency detected. ${resolutionStack.map((t) => t.name).join(' -> ')}`, - ); - } - // Add current type to resolution stack before resolving dependencies resolutionStack.push(type); @@ -96,9 +89,15 @@ class ContainerClass { } else { const paramTypes = (Reflect.getMetadata('design:paramtypes', type) ?? []) as Constructable[]; - const dependencies = paramTypes.map(

(paramType: Constructable

) => - this.get(paramType), - ); + + const dependencies = paramTypes.map(

(paramType: Constructable

, index: number) => { + if (paramType === undefined) { + throw new DIError( + `Circular dependency detected in ${type.name} at index ${index}.\n${resolutionStack.map((t) => t.name).join(' -> ')}\n`, + ); + } + return this.get(paramType); + }); // Create new instance with resolved dependencies instance = new (type as Constructable)(...dependencies) as T; } diff --git a/packages/@n8n/nodes-langchain/credentials/DeepSeekApi.credentials.ts b/packages/@n8n/nodes-langchain/credentials/DeepSeekApi.credentials.ts new file mode 100644 index 0000000000..6088fbbb81 --- /dev/null +++ b/packages/@n8n/nodes-langchain/credentials/DeepSeekApi.credentials.ts @@ -0,0 +1,47 @@ +import type { + IAuthenticateGeneric, + ICredentialTestRequest, + ICredentialType, + INodeProperties, +} from 'n8n-workflow'; + +export class DeepSeekApi implements ICredentialType { + name = 'deepSeekApi'; + + displayName = 'DeepSeek'; + + documentationUrl = 'deepseek'; + + properties: INodeProperties[] = [ + { + displayName: 'API Key', + name: 'apiKey', + type: 'string', + typeOptions: { password: true }, + required: true, + default: '', + }, + { + displayName: 'Base URL', + name: 'url', + type: 'hidden', + default: 'https://api.deepseek.com', + }, + ]; + + authenticate: IAuthenticateGeneric = { + type: 'generic', + properties: { + headers: { + Authorization: '=Bearer {{$credentials.apiKey}}', + }, + }, + }; + + test: ICredentialTestRequest = { + request: { + baseURL: '={{ $credentials.url }}', + url: '/models', + }, + }; +} diff --git a/packages/@n8n/nodes-langchain/credentials/OpenRouterApi.credentials.ts b/packages/@n8n/nodes-langchain/credentials/OpenRouterApi.credentials.ts new file mode 100644 index 0000000000..e21a84d020 --- /dev/null +++ b/packages/@n8n/nodes-langchain/credentials/OpenRouterApi.credentials.ts @@ -0,0 +1,47 @@ +import type { + IAuthenticateGeneric, + ICredentialTestRequest, + ICredentialType, + INodeProperties, +} from 'n8n-workflow'; + +export class OpenRouterApi implements ICredentialType { + name = 'openRouterApi'; + + displayName = 'OpenRouter'; + + documentationUrl = 'openrouter'; + + properties: INodeProperties[] = [ + { + displayName: 'API Key', + name: 'apiKey', + type: 'string', + typeOptions: { password: true }, + required: true, + default: '', + }, + { + displayName: 'Base URL', + name: 'url', + type: 'hidden', + default: 'https://openrouter.ai/api/v1', + }, + ]; + + authenticate: IAuthenticateGeneric = { + type: 'generic', + properties: { + headers: { + Authorization: '=Bearer {{$credentials.apiKey}}', + }, + }, + }; + + test: ICredentialTestRequest = { + request: { + baseURL: '={{ $credentials.url }}', + url: '/models', + }, + }; +} diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts index 230a6ddc6e..c5a46a1192 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/Agent.node.ts @@ -91,6 +91,8 @@ function getInputs( '@n8n/n8n-nodes-langchain.lmChatGoogleVertex', '@n8n/n8n-nodes-langchain.lmChatMistralCloud', '@n8n/n8n-nodes-langchain.lmChatAzureOpenAi', + '@n8n/n8n-nodes-langchain.lmChatDeepSeek', + '@n8n/n8n-nodes-langchain.lmChatOpenRouter', ], }, }, @@ -119,6 +121,8 @@ function getInputs( '@n8n/n8n-nodes-langchain.lmChatGroq', '@n8n/n8n-nodes-langchain.lmChatGoogleVertex', '@n8n/n8n-nodes-langchain.lmChatGoogleGemini', + '@n8n/n8n-nodes-langchain.lmChatDeepSeek', + '@n8n/n8n-nodes-langchain.lmChatOpenRouter', ], }, }, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.test.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.test.ts new file mode 100644 index 0000000000..fc67b6bb17 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.test.ts @@ -0,0 +1,156 @@ +import { mock } from 'jest-mock-extended'; +import type { PostgresNodeCredentials } from 'n8n-nodes-base/nodes/Postgres/v2/helpers/interfaces'; +import type { IExecuteFunctions } from 'n8n-workflow'; + +import { getPostgresDataSource } from './postgres'; + +describe('Postgres SSL settings', () => { + const credentials = mock({ + host: 'localhost', + port: 5432, + user: 'user', + password: 'password', + database: 'database', + }); + + test('ssl is disabled + allowUnauthorizedCerts is false', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'disable', + allowUnauthorizedCerts: false, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: false, + }); + }); + + test('ssl is disabled + allowUnauthorizedCerts is true', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'disable', + allowUnauthorizedCerts: true, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: false, + }); + }); + + test('ssl is disabled + allowUnauthorizedCerts is undefined', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'disable', + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: false, + }); + }); + + test('ssl is allow + allowUnauthorizedCerts is false', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'allow', + allowUnauthorizedCerts: false, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: true, + }); + }); + + test('ssl is allow + allowUnauthorizedCerts is true', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'allow', + allowUnauthorizedCerts: true, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: { rejectUnauthorized: false }, + }); + }); + + test('ssl is allow + allowUnauthorizedCerts is undefined', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'allow', + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: true, + }); + }); + + test('ssl is require + allowUnauthorizedCerts is false', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'require', + allowUnauthorizedCerts: false, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: true, + }); + }); + + test('ssl is require + allowUnauthorizedCerts is true', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'require', + allowUnauthorizedCerts: true, + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: { rejectUnauthorized: false }, + }); + }); + + test('ssl is require + allowUnauthorizedCerts is undefined', async () => { + const context = mock({ + getCredentials: jest.fn().mockReturnValue({ + ...credentials, + ssl: 'require', + }), + }); + + const dataSource = await getPostgresDataSource.call(context); + + expect(dataSource.options).toMatchObject({ + ssl: true, + }); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.ts index 31dda9ed72..8654dcafbb 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/other/handlers/postgres.ts @@ -1,29 +1,23 @@ import { DataSource } from '@n8n/typeorm'; +import type { PostgresNodeCredentials } from 'n8n-nodes-base/dist/nodes/Postgres/v2/helpers/interfaces'; import { type IExecuteFunctions } from 'n8n-workflow'; +import type { TlsOptions } from 'tls'; export async function getPostgresDataSource(this: IExecuteFunctions): Promise { - const credentials = await this.getCredentials('postgres'); + const credentials = await this.getCredentials('postgres'); - const dataSource = new DataSource({ - type: 'postgres', - host: credentials.host as string, - port: credentials.port as number, - username: credentials.user as string, - password: credentials.password as string, - database: credentials.database as string, - }); - - if (credentials.allowUnauthorizedCerts === true) { - dataSource.setOptions({ - ssl: { - rejectUnauthorized: true, - }, - }); - } else { - dataSource.setOptions({ - ssl: !['disable', undefined].includes(credentials.ssl as string | undefined), - }); + let ssl: TlsOptions | boolean = !['disable', undefined].includes(credentials.ssl); + if (credentials.allowUnauthorizedCerts && ssl) { + ssl = { rejectUnauthorized: false }; } - return dataSource; + return new DataSource({ + type: 'postgres', + host: credentials.host, + port: credentials.port, + username: credentials.user, + password: credentials.password, + database: credentials.database, + ssl, + }); } diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts index 4b2ddf5db9..45d3932309 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts @@ -34,6 +34,7 @@ import { getOptionalOutputParsers } from '@utils/output_parsers/N8nOutputParser' import { getTemplateNoticeField } from '@utils/sharedFields'; import { getTracingConfig } from '@utils/tracing'; +import { dataUriFromImageData, UnsupportedMimeTypeError } from './utils'; import { getCustomErrorMessage as getCustomOpenAiErrorMessage, isOpenAiError, @@ -88,21 +89,28 @@ async function getImageMessage( NodeConnectionType.AiLanguageModel, 0, )) as BaseLanguageModel; - const dataURI = `data:image/jpeg;base64,${bufferData.toString('base64')}`; - const directUriModels = [ChatGoogleGenerativeAI, ChatOllama]; - const imageUrl = directUriModels.some((i) => model instanceof i) - ? dataURI - : { url: dataURI, detail }; + try { + const dataURI = dataUriFromImageData(binaryData, bufferData); - return new HumanMessage({ - content: [ - { - type: 'image_url', - image_url: imageUrl, - }, - ], - }); + const directUriModels = [ChatGoogleGenerativeAI, ChatOllama]; + const imageUrl = directUriModels.some((i) => model instanceof i) + ? dataURI + : { url: dataURI, detail }; + + return new HumanMessage({ + content: [ + { + type: 'image_url', + image_url: imageUrl, + }, + ], + }); + } catch (error) { + if (error instanceof UnsupportedMimeTypeError) + throw new NodeOperationError(context.getNode(), error.message); + throw error; + } } async function getChainPromptTemplate( diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/test/utils.test.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/test/utils.test.ts new file mode 100644 index 0000000000..2207caf20b --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/test/utils.test.ts @@ -0,0 +1,23 @@ +import { mock } from 'jest-mock-extended'; +import type { IBinaryData } from 'n8n-workflow'; + +import { dataUriFromImageData, UnsupportedMimeTypeError } from '../utils'; + +describe('dataUriFromImageData', () => { + it('should not throw an error on images', async () => { + const mockBuffer = Buffer.from('Test data'); + const mockBinaryData = mock({ mimeType: 'image/jpeg' }); + + const dataUri = dataUriFromImageData(mockBinaryData, mockBuffer); + expect(dataUri).toBe('data:image/jpeg;base64,VGVzdCBkYXRh'); + }); + + it('should throw an UnsupportetMimeTypeError on non-images', async () => { + const mockBuffer = Buffer.from('Test data'); + const mockBinaryData = mock({ mimeType: 'text/plain' }); + + expect(() => { + dataUriFromImageData(mockBinaryData, mockBuffer); + }).toThrow(UnsupportedMimeTypeError); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/utils.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/utils.ts new file mode 100644 index 0000000000..95671d40fe --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/utils.ts @@ -0,0 +1,12 @@ +import type { IBinaryData } from 'n8n-workflow'; +import { ApplicationError } from 'n8n-workflow'; + +export class UnsupportedMimeTypeError extends ApplicationError {} + +export function dataUriFromImageData(binaryData: IBinaryData, bufferData: Buffer) { + if (!binaryData.mimeType?.startsWith('image/')) + throw new UnsupportedMimeTypeError( + `${binaryData.mimeType} is not a supported type of binary data. Only images are supported.`, + ); + return `data:${binaryData.mimeType};base64,${bufferData.toString('base64')}`; +} diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts index cd44cb114b..58d90ff90b 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts @@ -24,7 +24,7 @@ const modelParameter: INodeProperties = { routing: { request: { method: 'GET', - url: '={{ $parameter.options?.baseURL?.split("/").slice(-1).pop() || "v1" }}/models', + url: '={{ $parameter.options?.baseURL?.split("/").slice(-1).pop() || $credentials?.url?.split("/").slice(-1).pop() || "v1" }}/models', }, output: { postReceive: [ diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts index cf24d944de..1db5ef7706 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.ts @@ -11,18 +11,25 @@ import { import { getConnectionHintNoticeField } from '@utils/sharedFields'; +import { searchModels } from './methods/loadModels'; import { openAiFailedAttemptHandler } from '../../vendors/OpenAi/helpers/error-handling'; import { makeN8nLlmFailedAttemptHandler } from '../n8nLlmFailedAttemptHandler'; import { N8nLlmTracing } from '../N8nLlmTracing'; export class LmChatOpenAi implements INodeType { + methods = { + listSearch: { + searchModels, + }, + }; + description: INodeTypeDescription = { displayName: 'OpenAI Chat Model', // eslint-disable-next-line n8n-nodes-base/node-class-description-name-miscased name: 'lmChatOpenAi', icon: { light: 'file:openAiLight.svg', dark: 'file:openAiLight.dark.svg' }, group: ['transform'], - version: [1, 1.1], + version: [1, 1.1, 1.2], description: 'For advanced usage with an AI chain', defaults: { name: 'OpenAI Chat Model', @@ -101,6 +108,7 @@ export class LmChatOpenAi implements INodeType { ($credentials?.url && !$credentials.url.includes('api.openai.com')) || $responseItem.id.startsWith('ft:') || $responseItem.id.startsWith('o1') || + $responseItem.id.startsWith('o3') || ($responseItem.id.startsWith('gpt-') && !$responseItem.id.includes('instruct')) }}`, }, @@ -130,6 +138,42 @@ export class LmChatOpenAi implements INodeType { }, }, default: 'gpt-4o-mini', + displayOptions: { + hide: { + '@version': [{ _cnd: { gte: 1.2 } }], + }, + }, + }, + { + displayName: 'Model', + name: 'model', + type: 'resourceLocator', + default: { mode: 'list', value: 'gpt-4o-mini' }, + required: true, + modes: [ + { + displayName: 'From List', + name: 'list', + type: 'list', + placeholder: 'Select a model...', + typeOptions: { + searchListMethod: 'searchModels', + searchable: true, + }, + }, + { + displayName: 'ID', + name: 'id', + type: 'string', + placeholder: 'gpt-4o-mini', + }, + ], + description: 'The model. Choose from the list, or specify an ID.', + displayOptions: { + hide: { + '@version': [{ _cnd: { lte: 1.1 } }], + }, + }, }, { displayName: @@ -215,11 +259,43 @@ export class LmChatOpenAi implements INodeType { displayName: 'Sampling Temperature', name: 'temperature', default: 0.7, - typeOptions: { maxValue: 1, minValue: 0, numberPrecision: 1 }, + typeOptions: { maxValue: 2, minValue: 0, numberPrecision: 1 }, description: 'Controls randomness: Lowering results in less random completions. As the temperature approaches zero, the model will become deterministic and repetitive.', type: 'number', }, + { + displayName: 'Reasoning Effort', + name: 'reasoningEffort', + default: 'medium', + description: + 'Controls the amount of reasoning tokens to use. A value of "low" will favor speed and economical token usage, "high" will favor more complete reasoning at the cost of more tokens generated and slower responses.', + type: 'options', + options: [ + { + name: 'Low', + value: 'low', + description: 'Favors speed and economical token usage', + }, + { + name: 'Medium', + value: 'medium', + description: 'Balance between speed and reasoning accuracy', + }, + { + name: 'High', + value: 'high', + description: + 'Favors more complete reasoning at the cost of more tokens generated and slower responses', + }, + ], + displayOptions: { + show: { + // reasoning_effort is only available on o1, o1-versioned, or on o3-mini and beyond. Not on o1-mini or other GPT-models. + '/model': [{ _cnd: { regex: '(^o1([-\\d]+)?$)|(^o[3-9].*)' } }], + }, + }, + }, { displayName: 'Timeout', name: 'timeout', @@ -251,7 +327,12 @@ export class LmChatOpenAi implements INodeType { async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { const credentials = await this.getCredentials('openAiApi'); - const modelName = this.getNodeParameter('model', itemIndex) as string; + const version = this.getNode().typeVersion; + const modelName = + version >= 1.2 + ? (this.getNodeParameter('model.value', itemIndex) as string) + : (this.getNodeParameter('model', itemIndex) as string); + const options = this.getNodeParameter('options', itemIndex, {}) as { baseURL?: string; frequencyPenalty?: number; @@ -262,6 +343,7 @@ export class LmChatOpenAi implements INodeType { temperature?: number; topP?: number; responseFormat?: 'text' | 'json_object'; + reasoningEffort?: 'low' | 'medium' | 'high'; }; const configuration: ClientOptions = {}; @@ -271,6 +353,15 @@ export class LmChatOpenAi implements INodeType { configuration.baseURL = credentials.url as string; } + // Extra options to send to OpenAI, that are not directly supported by LangChain + const modelKwargs: { + response_format?: object; + reasoning_effort?: 'low' | 'medium' | 'high'; + } = {}; + if (options.responseFormat) modelKwargs.response_format = { type: options.responseFormat }; + if (options.reasoningEffort && ['low', 'medium', 'high'].includes(options.reasoningEffort)) + modelKwargs.reasoning_effort = options.reasoningEffort; + const model = new ChatOpenAI({ openAIApiKey: credentials.apiKey as string, modelName, @@ -279,11 +370,7 @@ export class LmChatOpenAi implements INodeType { maxRetries: options.maxRetries ?? 2, configuration, callbacks: [new N8nLlmTracing(this)], - modelKwargs: options.responseFormat - ? { - response_format: { type: options.responseFormat }, - } - : undefined, + modelKwargs, onFailedAttempt: makeN8nLlmFailedAttemptHandler(this, openAiFailedAttemptHandler), }); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts new file mode 100644 index 0000000000..b2f728678f --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts @@ -0,0 +1,112 @@ +import type { ILoadOptionsFunctions } from 'n8n-workflow'; +import OpenAI from 'openai'; + +import { searchModels } from '../loadModels'; + +jest.mock('openai'); + +describe('searchModels', () => { + let mockContext: jest.Mocked; + let mockOpenAI: jest.Mocked; + + beforeEach(() => { + mockContext = { + getCredentials: jest.fn().mockResolvedValue({ + apiKey: 'test-api-key', + }), + getNodeParameter: jest.fn().mockReturnValue(''), + } as unknown as jest.Mocked; + + // Setup OpenAI mock with required properties + const mockOpenAIInstance = { + apiKey: 'test-api-key', + organization: null, + project: null, + _options: {}, + models: { + list: jest.fn().mockResolvedValue({ + data: [ + { id: 'gpt-4' }, + { id: 'gpt-3.5-turbo' }, + { id: 'gpt-3.5-turbo-instruct' }, + { id: 'ft:gpt-3.5-turbo' }, + { id: 'o1-model' }, + { id: 'other-model' }, + ], + }), + }, + } as unknown as OpenAI; + + (OpenAI as jest.MockedClass).mockImplementation(() => mockOpenAIInstance); + + mockOpenAI = OpenAI as jest.Mocked; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should return filtered models if custom API endpoint is not provided', async () => { + const result = await searchModels.call(mockContext); + + expect(mockOpenAI).toHaveBeenCalledWith({ + baseURL: 'https://api.openai.com/v1', + apiKey: 'test-api-key', + }); + expect(result.results).toHaveLength(4); + }); + + it('should initialize OpenAI with correct credentials', async () => { + mockContext.getCredentials.mockResolvedValueOnce({ + apiKey: 'test-api-key', + url: 'https://test-url.com', + }); + await searchModels.call(mockContext); + + expect(mockOpenAI).toHaveBeenCalledWith({ + baseURL: 'https://test-url.com', + apiKey: 'test-api-key', + }); + }); + + it('should use default OpenAI URL if no custom URL provided', async () => { + mockContext.getCredentials = jest.fn().mockResolvedValue({ + apiKey: 'test-api-key', + }); + + await searchModels.call(mockContext); + + expect(mockOpenAI).toHaveBeenCalledWith({ + baseURL: 'https://api.openai.com/v1', + apiKey: 'test-api-key', + }); + }); + + it('should include all models for custom API endpoints', async () => { + mockContext.getNodeParameter = jest.fn().mockReturnValue('https://custom-api.com'); + + const result = await searchModels.call(mockContext); + + expect(result.results).toHaveLength(6); + }); + + it('should filter models based on search term', async () => { + const result = await searchModels.call(mockContext, 'gpt'); + + expect(result.results).toEqual([ + { name: 'gpt-4', value: 'gpt-4' }, + { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, + { name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' }, + ]); + }); + + it('should handle case-insensitive search', async () => { + const result = await searchModels.call(mockContext, 'GPT'); + + expect(result.results).toEqual([ + { name: 'gpt-4', value: 'gpt-4' }, + { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, + { name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' }, + ]); + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts new file mode 100644 index 0000000000..bfcbfe1c30 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts @@ -0,0 +1,38 @@ +import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow'; +import OpenAI from 'openai'; + +export async function searchModels( + this: ILoadOptionsFunctions, + filter?: string, +): Promise { + const credentials = await this.getCredentials('openAiApi'); + const baseURL = + (this.getNodeParameter('options.baseURL', '') as string) || + (credentials.url as string) || + 'https://api.openai.com/v1'; + + const openai = new OpenAI({ baseURL, apiKey: credentials.apiKey as string }); + const { data: models = [] } = await openai.models.list(); + + const filteredModels = models.filter((model: { id: string }) => { + const isValidModel = + (baseURL && !baseURL.includes('api.openai.com')) || + model.id.startsWith('ft:') || + model.id.startsWith('o1') || + model.id.startsWith('o3') || + (model.id.startsWith('gpt-') && !model.id.includes('instruct')); + + if (!filter) return isValidModel; + + return isValidModel && model.id.toLowerCase().includes(filter.toLowerCase()); + }); + + const results = { + results: filteredModels.map((model: { id: string }) => ({ + name: model.id, + value: model.id, + })), + }; + + return results; +} diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/LmChatDeepSeek.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/LmChatDeepSeek.node.ts new file mode 100644 index 0000000000..bf811ac2fe --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/LmChatDeepSeek.node.ts @@ -0,0 +1,253 @@ +/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ + +import { ChatOpenAI, type ClientOptions } from '@langchain/openai'; +import { + NodeConnectionType, + type INodeType, + type INodeTypeDescription, + type ISupplyDataFunctions, + type SupplyData, +} from 'n8n-workflow'; + +import { getConnectionHintNoticeField } from '@utils/sharedFields'; + +import { openAiFailedAttemptHandler } from '../../vendors/OpenAi/helpers/error-handling'; +import { makeN8nLlmFailedAttemptHandler } from '../n8nLlmFailedAttemptHandler'; +import { N8nLlmTracing } from '../N8nLlmTracing'; + +export class LmChatDeepSeek implements INodeType { + description: INodeTypeDescription = { + displayName: 'DeepSeek Chat Model', + // eslint-disable-next-line n8n-nodes-base/node-class-description-name-miscased + name: 'lmChatDeepSeek', + icon: 'file:deepseek.svg', + group: ['transform'], + version: [1], + description: 'For advanced usage with an AI chain', + defaults: { + name: 'DeepSeek Chat Model', + }, + codex: { + categories: ['AI'], + subcategories: { + AI: ['Language Models', 'Root Nodes'], + 'Language Models': ['Chat Models (Recommended)'], + }, + resources: { + primaryDocumentation: [ + { + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.lmchatdeepseek/', + }, + ], + }, + }, + // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node + inputs: [], + // eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong + outputs: [NodeConnectionType.AiLanguageModel], + outputNames: ['Model'], + credentials: [ + { + name: 'deepSeekApi', + required: true, + }, + ], + requestDefaults: { + ignoreHttpStatusErrors: true, + baseURL: '={{ $credentials?.url }}', + }, + properties: [ + getConnectionHintNoticeField([NodeConnectionType.AiChain, NodeConnectionType.AiAgent]), + { + displayName: + 'If using JSON response format, you must include word "json" in the prompt in your chain or agent. Also, make sure to select latest models released post November 2023.', + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + '/options.responseFormat': ['json_object'], + }, + }, + }, + { + displayName: 'Model', + name: 'model', + type: 'options', + description: + 'The model which will generate the completion. Learn more.', + typeOptions: { + loadOptions: { + routing: { + request: { + method: 'GET', + url: '/models', + }, + output: { + postReceive: [ + { + type: 'rootProperty', + properties: { + property: 'data', + }, + }, + { + type: 'setKeyValue', + properties: { + name: '={{$responseItem.id}}', + value: '={{$responseItem.id}}', + }, + }, + { + type: 'sort', + properties: { + key: 'name', + }, + }, + ], + }, + }, + }, + }, + routing: { + send: { + type: 'body', + property: 'model', + }, + }, + default: 'deepseek-chat', + }, + { + displayName: 'Options', + name: 'options', + placeholder: 'Add Option', + description: 'Additional options to add', + type: 'collection', + default: {}, + options: [ + { + displayName: 'Frequency Penalty', + name: 'frequencyPenalty', + default: 0, + typeOptions: { maxValue: 2, minValue: -2, numberPrecision: 1 }, + description: + "Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim", + type: 'number', + }, + { + displayName: 'Maximum Number of Tokens', + name: 'maxTokens', + default: -1, + description: + 'The maximum number of tokens to generate in the completion. Most models have a context length of 2048 tokens (except for the newest models, which support 32,768).', + type: 'number', + typeOptions: { + maxValue: 32768, + }, + }, + { + displayName: 'Response Format', + name: 'responseFormat', + default: 'text', + type: 'options', + options: [ + { + name: 'Text', + value: 'text', + description: 'Regular text response', + }, + { + name: 'JSON', + value: 'json_object', + description: + 'Enables JSON mode, which should guarantee the message the model generates is valid JSON', + }, + ], + }, + { + displayName: 'Presence Penalty', + name: 'presencePenalty', + default: 0, + typeOptions: { maxValue: 2, minValue: -2, numberPrecision: 1 }, + description: + "Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics", + type: 'number', + }, + { + displayName: 'Sampling Temperature', + name: 'temperature', + default: 0.7, + typeOptions: { maxValue: 2, minValue: 0, numberPrecision: 1 }, + description: + 'Controls randomness: Lowering results in less random completions. As the temperature approaches zero, the model will become deterministic and repetitive.', + type: 'number', + }, + { + displayName: 'Timeout', + name: 'timeout', + default: 360000, + description: 'Maximum amount of time a request is allowed to take in milliseconds', + type: 'number', + }, + { + displayName: 'Max Retries', + name: 'maxRetries', + default: 2, + description: 'Maximum number of retries to attempt', + type: 'number', + }, + { + displayName: 'Top P', + name: 'topP', + default: 1, + typeOptions: { maxValue: 1, minValue: 0, numberPrecision: 1 }, + description: + 'Controls diversity via nucleus sampling: 0.5 means half of all likelihood-weighted options are considered. We generally recommend altering this or temperature but not both.', + type: 'number', + }, + ], + }, + ], + }; + + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { + const credentials = await this.getCredentials('deepSeekApi'); + + const modelName = this.getNodeParameter('model', itemIndex) as string; + + const options = this.getNodeParameter('options', itemIndex, {}) as { + frequencyPenalty?: number; + maxTokens?: number; + maxRetries: number; + timeout: number; + presencePenalty?: number; + temperature?: number; + topP?: number; + responseFormat?: 'text' | 'json_object'; + }; + + const configuration: ClientOptions = { + baseURL: credentials.url, + }; + + const model = new ChatOpenAI({ + openAIApiKey: credentials.apiKey, + modelName, + ...options, + timeout: options.timeout ?? 60000, + maxRetries: options.maxRetries ?? 2, + configuration, + callbacks: [new N8nLlmTracing(this)], + modelKwargs: options.responseFormat + ? { + response_format: { type: options.responseFormat }, + } + : undefined, + onFailedAttempt: makeN8nLlmFailedAttemptHandler(this, openAiFailedAttemptHandler), + }); + + return { + response: model, + }; + } +} diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/deepseek.svg b/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/deepseek.svg new file mode 100644 index 0000000000..3395016ce0 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatDeepSeek/deepseek.svg @@ -0,0 +1 @@ +DeepSeek diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts index f8b7d2bb3e..f6ff0f5038 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.ts @@ -1,12 +1,13 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import type { SafetySetting } from '@google/generative-ai'; import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; -import { - NodeConnectionType, - type INodeType, - type INodeTypeDescription, - type ISupplyDataFunctions, - type SupplyData, +import { NodeConnectionType } from 'n8n-workflow'; +import type { + NodeError, + INodeType, + INodeTypeDescription, + ISupplyDataFunctions, + SupplyData, } from 'n8n-workflow'; import { getConnectionHintNoticeField } from '@utils/sharedFields'; @@ -15,6 +16,13 @@ import { additionalOptions } from '../gemini-common/additional-options'; import { makeN8nLlmFailedAttemptHandler } from '../n8nLlmFailedAttemptHandler'; import { N8nLlmTracing } from '../N8nLlmTracing'; +function errorDescriptionMapper(error: NodeError) { + if (error.description?.includes('properties: should be non-empty for OBJECT type')) { + return 'Google Gemini requires at least one dynamic parameter when using tools'; + } + + return error.description ?? 'Unknown error'; +} export class LmChatGoogleGemini implements INodeType { description: INodeTypeDescription = { displayName: 'Google Gemini Chat Model', @@ -140,13 +148,14 @@ export class LmChatGoogleGemini implements INodeType { const model = new ChatGoogleGenerativeAI({ apiKey: credentials.apiKey as string, + baseUrl: credentials.host as string, modelName, topK: options.topK, topP: options.topP, temperature: options.temperature, maxOutputTokens: options.maxOutputTokens, safetySettings, - callbacks: [new N8nLlmTracing(this)], + callbacks: [new N8nLlmTracing(this, { errorDescriptionMapper })], onFailedAttempt: makeN8nLlmFailedAttemptHandler(this), }); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/LmChatOpenRouter.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/LmChatOpenRouter.node.ts new file mode 100644 index 0000000000..57a14028e7 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/LmChatOpenRouter.node.ts @@ -0,0 +1,252 @@ +/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ + +import { ChatOpenAI, type ClientOptions } from '@langchain/openai'; +import { + NodeConnectionType, + type INodeType, + type INodeTypeDescription, + type ISupplyDataFunctions, + type SupplyData, +} from 'n8n-workflow'; + +import { getConnectionHintNoticeField } from '@utils/sharedFields'; + +import { openAiFailedAttemptHandler } from '../../vendors/OpenAi/helpers/error-handling'; +import { makeN8nLlmFailedAttemptHandler } from '../n8nLlmFailedAttemptHandler'; +import { N8nLlmTracing } from '../N8nLlmTracing'; + +export class LmChatOpenRouter implements INodeType { + description: INodeTypeDescription = { + displayName: 'OpenRouter Chat Model', + name: 'lmChatOpenRouter', + icon: { light: 'file:openrouter.svg', dark: 'file:openrouter.dark.svg' }, + group: ['transform'], + version: [1], + description: 'For advanced usage with an AI chain', + defaults: { + name: 'OpenRouter Chat Model', + }, + codex: { + categories: ['AI'], + subcategories: { + AI: ['Language Models', 'Root Nodes'], + 'Language Models': ['Chat Models (Recommended)'], + }, + resources: { + primaryDocumentation: [ + { + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.lmchatopenrouter/', + }, + ], + }, + }, + // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node + inputs: [], + // eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong + outputs: [NodeConnectionType.AiLanguageModel], + outputNames: ['Model'], + credentials: [ + { + name: 'openRouterApi', + required: true, + }, + ], + requestDefaults: { + ignoreHttpStatusErrors: true, + baseURL: '={{ $credentials?.url }}', + }, + properties: [ + getConnectionHintNoticeField([NodeConnectionType.AiChain, NodeConnectionType.AiAgent]), + { + displayName: + 'If using JSON response format, you must include word "json" in the prompt in your chain or agent. Also, make sure to select latest models released post November 2023.', + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + '/options.responseFormat': ['json_object'], + }, + }, + }, + { + displayName: 'Model', + name: 'model', + type: 'options', + description: + 'The model which will generate the completion. Learn more.', + typeOptions: { + loadOptions: { + routing: { + request: { + method: 'GET', + url: '/models', + }, + output: { + postReceive: [ + { + type: 'rootProperty', + properties: { + property: 'data', + }, + }, + { + type: 'setKeyValue', + properties: { + name: '={{$responseItem.id}}', + value: '={{$responseItem.id}}', + }, + }, + { + type: 'sort', + properties: { + key: 'name', + }, + }, + ], + }, + }, + }, + }, + routing: { + send: { + type: 'body', + property: 'model', + }, + }, + default: 'openai/gpt-4o-mini', + }, + { + displayName: 'Options', + name: 'options', + placeholder: 'Add Option', + description: 'Additional options to add', + type: 'collection', + default: {}, + options: [ + { + displayName: 'Frequency Penalty', + name: 'frequencyPenalty', + default: 0, + typeOptions: { maxValue: 2, minValue: -2, numberPrecision: 1 }, + description: + "Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim", + type: 'number', + }, + { + displayName: 'Maximum Number of Tokens', + name: 'maxTokens', + default: -1, + description: + 'The maximum number of tokens to generate in the completion. Most models have a context length of 2048 tokens (except for the newest models, which support 32,768).', + type: 'number', + typeOptions: { + maxValue: 32768, + }, + }, + { + displayName: 'Response Format', + name: 'responseFormat', + default: 'text', + type: 'options', + options: [ + { + name: 'Text', + value: 'text', + description: 'Regular text response', + }, + { + name: 'JSON', + value: 'json_object', + description: + 'Enables JSON mode, which should guarantee the message the model generates is valid JSON', + }, + ], + }, + { + displayName: 'Presence Penalty', + name: 'presencePenalty', + default: 0, + typeOptions: { maxValue: 2, minValue: -2, numberPrecision: 1 }, + description: + "Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics", + type: 'number', + }, + { + displayName: 'Sampling Temperature', + name: 'temperature', + default: 0.7, + typeOptions: { maxValue: 2, minValue: 0, numberPrecision: 1 }, + description: + 'Controls randomness: Lowering results in less random completions. As the temperature approaches zero, the model will become deterministic and repetitive.', + type: 'number', + }, + { + displayName: 'Timeout', + name: 'timeout', + default: 360000, + description: 'Maximum amount of time a request is allowed to take in milliseconds', + type: 'number', + }, + { + displayName: 'Max Retries', + name: 'maxRetries', + default: 2, + description: 'Maximum number of retries to attempt', + type: 'number', + }, + { + displayName: 'Top P', + name: 'topP', + default: 1, + typeOptions: { maxValue: 1, minValue: 0, numberPrecision: 1 }, + description: + 'Controls diversity via nucleus sampling: 0.5 means half of all likelihood-weighted options are considered. We generally recommend altering this or temperature but not both.', + type: 'number', + }, + ], + }, + ], + }; + + async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { + const credentials = await this.getCredentials('openRouterApi'); + + const modelName = this.getNodeParameter('model', itemIndex) as string; + + const options = this.getNodeParameter('options', itemIndex, {}) as { + frequencyPenalty?: number; + maxTokens?: number; + maxRetries: number; + timeout: number; + presencePenalty?: number; + temperature?: number; + topP?: number; + responseFormat?: 'text' | 'json_object'; + }; + + const configuration: ClientOptions = { + baseURL: credentials.url, + }; + + const model = new ChatOpenAI({ + openAIApiKey: credentials.apiKey, + modelName, + ...options, + timeout: options.timeout ?? 60000, + maxRetries: options.maxRetries ?? 2, + configuration, + callbacks: [new N8nLlmTracing(this)], + modelKwargs: options.responseFormat + ? { + response_format: { type: options.responseFormat }, + } + : undefined, + onFailedAttempt: makeN8nLlmFailedAttemptHandler(this, openAiFailedAttemptHandler), + }); + + return { + response: model, + }; + } +} diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.dark.svg b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.dark.svg new file mode 100644 index 0000000000..0b8bb9df1b --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.dark.svg @@ -0,0 +1 @@ +OpenRouter diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.svg b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.svg new file mode 100644 index 0000000000..749e44df71 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatOpenRouter/openrouter.svg @@ -0,0 +1 @@ +OpenRouter diff --git a/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts b/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts index 3d426309b7..7f7d70841e 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts @@ -61,11 +61,15 @@ export class N8nLlmTracing extends BaseCallbackHandler { totalTokens: completionTokens + promptTokens, }; }, + errorDescriptionMapper: (error: NodeError) => error.description, }; constructor( private executionFunctions: ISupplyDataFunctions, - options?: { tokensUsageParser: TokensUsageParser }, + options?: { + tokensUsageParser?: TokensUsageParser; + errorDescriptionMapper?: (error: NodeError) => string; + }, ) { super(); this.options = { ...this.options, ...options }; @@ -192,6 +196,10 @@ export class N8nLlmTracing extends BaseCallbackHandler { } if (error instanceof NodeError) { + if (this.options.errorDescriptionMapper) { + error.description = this.options.errorDescriptionMapper(error); + } + this.executionFunctions.addOutputData(this.connectionType, runDetails.index, error); } else { // If the error is not a NodeError, we wrap it in a NodeOperationError diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts index 18fd76e3c5..81be577f79 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryPostgresChat/MemoryPostgresChat.node.ts @@ -1,9 +1,9 @@ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */ import { PostgresChatMessageHistory } from '@langchain/community/stores/message/postgres'; import { BufferMemory, BufferWindowMemory } from 'langchain/memory'; +import { configurePostgres } from 'n8n-nodes-base/dist/nodes/Postgres/transport'; import type { PostgresNodeCredentials } from 'n8n-nodes-base/dist/nodes/Postgres/v2/helpers/interfaces'; import { postgresConnectionTest } from 'n8n-nodes-base/dist/nodes/Postgres/v2/methods/credentialTest'; -import { configurePostgres } from 'n8n-nodes-base/dist/nodes/Postgres/v2/transport'; import type { ISupplyDataFunctions, INodeType, @@ -115,12 +115,7 @@ export class MemoryPostgresChat implements INodeType { ...kOptions, }); - async function closeFunction() { - void pool.end(); - } - return { - closeFunction, response: logWrapper(memory, this), }; } diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts index 05ed1e619c..346234c8d3 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/test/ToolHttpRequest.node.test.ts @@ -237,6 +237,69 @@ describe('ToolHttpRequest', () => { }), ); }); + + it('should return the error when receiving text that contains a null character', async () => { + helpers.httpRequest.mockResolvedValue({ + body: 'Hello\0World', + headers: { + 'content-type': 'text/plain', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/text/plain'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + // Check that the returned string is formatted as an error message. + expect(res).toContain('error'); + expect(res).toContain('Binary data is not supported'); + }); + + it('should return the error when receiving a JSON response containing a null character', async () => { + // Provide a raw JSON string with a literal null character. + helpers.httpRequest.mockResolvedValue({ + body: '{"message":"hello\0world"}', + headers: { + 'content-type': 'application/json', + }, + }); + + executeFunctions.getNodeParameter.mockImplementation((paramName: string) => { + switch (paramName) { + case 'method': + return 'GET'; + case 'url': + return 'https://httpbin.org/json'; + case 'options': + return {}; + case 'placeholderDefinitions.values': + return []; + default: + return undefined; + } + }); + + const { response } = await httpTool.supplyData.call(executeFunctions, 0); + const res = await (response as N8nTool).invoke({}); + expect(helpers.httpRequest).toHaveBeenCalled(); + // Check that the tool returns an error string rather than resolving to valid JSON. + expect(res).toContain('error'); + expect(res).toContain('Binary data is not supported'); + }); }); describe('Optimize response', () => { diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts index f1d6dfd150..0bd1b1a8a6 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts @@ -5,7 +5,6 @@ import { JSDOM } from 'jsdom'; import get from 'lodash/get'; import set from 'lodash/set'; import unset from 'lodash/unset'; -import * as mime from 'mime-types'; import { getOAuth2AdditionalParameters } from 'n8n-nodes-base/dist/nodes/HttpRequest/GenericFunctions'; import type { IDataObject, @@ -146,6 +145,25 @@ const defaultOptimizer = (response: T) => { return String(response); }; +function isBinary(data: unknown) { + // Check if data is a Buffer + if (Buffer.isBuffer(data)) { + return true; + } + + // If data is a string, assume it's text unless it contains null characters. + if (typeof data === 'string') { + // If the string contains a null character, it's likely binary. + if (data.includes('\0')) { + return true; + } + return false; + } + + // For any other type, assume it's not binary. + return false; +} + const htmlOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number, maxLength: number) => { const cssSelector = ctx.getNodeParameter('cssSelector', itemIndex, '') as string; const onlyContent = ctx.getNodeParameter('onlyContent', itemIndex, false) as boolean; @@ -755,13 +773,8 @@ export const configureToolFunction = ( if (!response) { try { // Check if the response is binary data - if (fullResponse?.headers?.['content-type']) { - const contentType = fullResponse.headers['content-type'] as string; - const mimeType = contentType.split(';')[0].trim(); - - if (mime.charset(mimeType) !== 'UTF-8') { - throw new NodeOperationError(ctx.getNode(), 'Binary data is not supported'); - } + if (fullResponse.body && isBinary(fullResponse.body)) { + throw new NodeOperationError(ctx.getNode(), 'Binary data is not supported'); } response = optimizeResponse(fullResponse.body); diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.node.ts index 22ca31e4da..98ca94cb1f 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.node.ts @@ -1,4 +1,3 @@ -import { loadWorkflowInputMappings } from 'n8n-nodes-base/dist/utils/workflowInputsResourceMapping/GenericFunctions'; import type { INodeTypeBaseDescription, ISupplyDataFunctions, @@ -7,6 +6,7 @@ import type { INodeTypeDescription, } from 'n8n-workflow'; +import { localResourceMapping } from './methods'; import { WorkflowToolService } from './utils/WorkflowToolService'; import { versionDescription } from './versionDescription'; @@ -21,9 +21,7 @@ export class ToolWorkflowV2 implements INodeType { } methods = { - localResourceMapping: { - loadWorkflowInputMappings, - }, + localResourceMapping, }; async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise { diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.test.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.test.ts index 73aa24c6b7..a5aa4e41bc 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/ToolWorkflowV2.test.ts @@ -11,9 +11,14 @@ import type { import { WorkflowToolService } from './utils/WorkflowToolService'; +type ISupplyDataFunctionsWithRunIndex = ISupplyDataFunctions & { runIndex: number }; + // Mock ISupplyDataFunctions interface -function createMockContext(overrides?: Partial): ISupplyDataFunctions { +function createMockContext( + overrides?: Partial, +): ISupplyDataFunctionsWithRunIndex { return { + runIndex: 0, getNodeParameter: jest.fn(), getWorkflowDataProxy: jest.fn(), getNode: jest.fn(), @@ -35,11 +40,11 @@ function createMockContext(overrides?: Partial): ISupplyDa warn: jest.fn(), }, ...overrides, - } as ISupplyDataFunctions; + } as ISupplyDataFunctionsWithRunIndex; } describe('WorkflowTool::WorkflowToolService', () => { - let context: ISupplyDataFunctions; + let context: ISupplyDataFunctionsWithRunIndex; let service: WorkflowToolService; beforeEach(() => { @@ -93,6 +98,7 @@ describe('WorkflowTool::WorkflowToolService', () => { expect(result).toBe(JSON.stringify(TEST_RESPONSE, null, 2)); expect(context.addOutputData).toHaveBeenCalled(); + expect(context.runIndex).toBe(1); }); it('should handle errors during tool execution', async () => { diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/index.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/index.ts new file mode 100644 index 0000000000..f43c9557ea --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/index.ts @@ -0,0 +1 @@ +export * as localResourceMapping from './localResourceMapping'; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/localResourceMapping.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/localResourceMapping.ts new file mode 100644 index 0000000000..74de9baae2 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/methods/localResourceMapping.ts @@ -0,0 +1,24 @@ +import { loadWorkflowInputMappings } from 'n8n-nodes-base/dist/utils/workflowInputsResourceMapping/GenericFunctions'; +import type { ILocalLoadOptionsFunctions, ResourceMapperFields } from 'n8n-workflow'; + +export async function loadSubWorkflowInputs( + this: ILocalLoadOptionsFunctions, +): Promise { + const { fields, subworkflowInfo, dataMode } = await loadWorkflowInputMappings.bind(this)(); + let emptyFieldsNotice: string | undefined; + if (fields.length === 0) { + const subworkflowLink = subworkflowInfo?.id + ? `sub-workflow’s trigger` + : 'sub-workflow’s trigger'; + + switch (dataMode) { + case 'passthrough': + emptyFieldsNotice = `This sub-workflow is set up to receive all input data, without specific inputs the Agent will not be able to pass data to this tool. You can define specific inputs in the ${subworkflowLink}.`; + break; + default: + emptyFieldsNotice = `This sub-workflow will not receive any input when called by your AI node. Define your expected input in the ${subworkflowLink}.`; + break; + } + } + return { fields, emptyFieldsNotice }; +} diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/FromAIParser.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/FromAIParser.ts deleted file mode 100644 index 4b9b6ed58e..0000000000 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/FromAIParser.ts +++ /dev/null @@ -1,284 +0,0 @@ -import type { ISupplyDataFunctions } from 'n8n-workflow'; -import { jsonParse, NodeOperationError } from 'n8n-workflow'; -import { z } from 'zod'; - -type AllowedTypes = 'string' | 'number' | 'boolean' | 'json'; -export interface FromAIArgument { - key: string; - description?: string; - type?: AllowedTypes; - defaultValue?: string | number | boolean | Record; -} - -// TODO: We copied this class from the core package, once the new nodes context work is merged, this should be available in root node context and this file can be removed. -// Please apply any changes to both files - -/** - * AIParametersParser - * - * This class encapsulates the logic for parsing node parameters, extracting $fromAI calls, - * generating Zod schemas, and creating LangChain tools. - */ -export class AIParametersParser { - private ctx: ISupplyDataFunctions; - - /** - * Constructs an instance of AIParametersParser. - * @param ctx The execution context. - */ - constructor(ctx: ISupplyDataFunctions) { - this.ctx = ctx; - } - - /** - * Generates a Zod schema based on the provided FromAIArgument placeholder. - * @param placeholder The FromAIArgument object containing key, type, description, and defaultValue. - * @returns A Zod schema corresponding to the placeholder's type and constraints. - */ - generateZodSchema(placeholder: FromAIArgument): z.ZodTypeAny { - let schema: z.ZodTypeAny; - - switch (placeholder.type?.toLowerCase()) { - case 'string': - schema = z.string(); - break; - case 'number': - schema = z.number(); - break; - case 'boolean': - schema = z.boolean(); - break; - case 'json': - schema = z.record(z.any()); - break; - default: - schema = z.string(); - } - - if (placeholder.description) { - schema = schema.describe(`${schema.description ?? ''} ${placeholder.description}`.trim()); - } - - if (placeholder.defaultValue !== undefined) { - schema = schema.default(placeholder.defaultValue); - } - - return schema; - } - - /** - * Recursively traverses the nodeParameters object to find all $fromAI calls. - * @param payload The current object or value being traversed. - * @param collectedArgs The array collecting FromAIArgument objects. - */ - traverseNodeParameters(payload: unknown, collectedArgs: FromAIArgument[]) { - if (typeof payload === 'string') { - const fromAICalls = this.extractFromAICalls(payload); - fromAICalls.forEach((call) => collectedArgs.push(call)); - } else if (Array.isArray(payload)) { - payload.forEach((item: unknown) => this.traverseNodeParameters(item, collectedArgs)); - } else if (typeof payload === 'object' && payload !== null) { - Object.values(payload).forEach((value) => this.traverseNodeParameters(value, collectedArgs)); - } - } - - /** - * Extracts all $fromAI calls from a given string - * @param str The string to search for $fromAI calls. - * @returns An array of FromAIArgument objects. - * - * This method uses a regular expression to find the start of each $fromAI function call - * in the input string. It then employs a character-by-character parsing approach to - * accurately extract the arguments of each call, handling nested parentheses and quoted strings. - * - * The parsing process: - * 1. Finds the starting position of a $fromAI call using regex. - * 2. Iterates through characters, keeping track of parentheses depth and quote status. - * 3. Handles escaped characters within quotes to avoid premature quote closing. - * 4. Builds the argument string until the matching closing parenthesis is found. - * 5. Parses the extracted argument string into a FromAIArgument object. - * 6. Repeats the process for all $fromAI calls in the input string. - * - */ - extractFromAICalls(str: string): FromAIArgument[] { - const args: FromAIArgument[] = []; - // Regular expression to match the start of a $fromAI function call - const pattern = /\$fromAI\s*\(\s*/gi; - let match: RegExpExecArray | null; - - while ((match = pattern.exec(str)) !== null) { - const startIndex = match.index + match[0].length; - let current = startIndex; - let inQuotes = false; - let quoteChar = ''; - let parenthesesCount = 1; - let argsString = ''; - - // Parse the arguments string, handling nested parentheses and quotes - while (current < str.length && parenthesesCount > 0) { - const char = str[current]; - - if (inQuotes) { - // Handle characters inside quotes, including escaped characters - if (char === '\\' && current + 1 < str.length) { - argsString += char + str[current + 1]; - current += 2; - continue; - } - - if (char === quoteChar) { - inQuotes = false; - quoteChar = ''; - } - argsString += char; - } else { - // Handle characters outside quotes - if (['"', "'", '`'].includes(char)) { - inQuotes = true; - quoteChar = char; - } else if (char === '(') { - parenthesesCount++; - } else if (char === ')') { - parenthesesCount--; - } - - // Only add characters if we're still inside the main parentheses - if (parenthesesCount > 0 || char !== ')') { - argsString += char; - } - } - - current++; - } - - // If parentheses are balanced, parse the arguments - if (parenthesesCount === 0) { - try { - const parsedArgs = this.parseArguments(argsString); - args.push(parsedArgs); - } catch (error) { - // If parsing fails, throw an ApplicationError with details - throw new NodeOperationError( - this.ctx.getNode(), - `Failed to parse $fromAI arguments: ${argsString}: ${error}`, - ); - } - } else { - // Log an error if parentheses are unbalanced - throw new NodeOperationError( - this.ctx.getNode(), - `Unbalanced parentheses while parsing $fromAI call: ${str.slice(startIndex)}`, - ); - } - } - - return args; - } - - /** - * Parses the arguments of a single $fromAI function call. - * @param argsString The string containing the function arguments. - * @returns A FromAIArgument object. - */ - parseArguments(argsString: string): FromAIArgument { - // Split arguments by commas not inside quotes - const args: string[] = []; - let currentArg = ''; - let inQuotes = false; - let quoteChar = ''; - let escapeNext = false; - - for (let i = 0; i < argsString.length; i++) { - const char = argsString[i]; - - if (escapeNext) { - currentArg += char; - escapeNext = false; - continue; - } - - if (char === '\\') { - escapeNext = true; - continue; - } - - if (['"', "'", '`'].includes(char)) { - if (!inQuotes) { - inQuotes = true; - quoteChar = char; - currentArg += char; - } else if (char === quoteChar) { - inQuotes = false; - quoteChar = ''; - currentArg += char; - } else { - currentArg += char; - } - continue; - } - - if (char === ',' && !inQuotes) { - args.push(currentArg.trim()); - currentArg = ''; - continue; - } - - currentArg += char; - } - - if (currentArg) { - args.push(currentArg.trim()); - } - - // Remove surrounding quotes if present - const cleanArgs = args.map((arg) => { - const trimmed = arg.trim(); - if ( - (trimmed.startsWith("'") && trimmed.endsWith("'")) || - (trimmed.startsWith('`') && trimmed.endsWith('`')) || - (trimmed.startsWith('"') && trimmed.endsWith('"')) - ) { - return trimmed - .slice(1, -1) - .replace(/\\'/g, "'") - .replace(/\\`/g, '`') - .replace(/\\"/g, '"') - .replace(/\\\\/g, '\\'); - } - return trimmed; - }); - - const type = cleanArgs?.[2] || 'string'; - - if (!['string', 'number', 'boolean', 'json'].includes(type.toLowerCase())) { - throw new NodeOperationError(this.ctx.getNode(), `Invalid type: ${type}`); - } - - return { - key: cleanArgs[0] || '', - description: cleanArgs[1], - type: (cleanArgs?.[2] ?? 'string') as AllowedTypes, - defaultValue: this.parseDefaultValue(cleanArgs[3]), - }; - } - - /** - * Parses the default value, preserving its original type. - * @param value The default value as a string. - * @returns The parsed default value in its appropriate type. - */ - parseDefaultValue( - value: string | undefined, - ): string | number | boolean | Record | undefined { - if (value === undefined || value === '') return undefined; - const lowerValue = value.toLowerCase(); - if (lowerValue === 'true') return true; - if (lowerValue === 'false') return false; - if (!isNaN(Number(value))) return Number(value); - try { - return jsonParse(value); - } catch { - return value; - } - } -} diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/WorkflowToolService.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/WorkflowToolService.ts index 2ce3c43556..3930938370 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/WorkflowToolService.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/WorkflowToolService.ts @@ -8,6 +8,7 @@ import { getCurrentWorkflowInputData } from 'n8n-nodes-base/dist/utils/workflowI import type { ExecuteWorkflowData, ExecutionError, + FromAIArgument, IDataObject, IExecuteWorkflowInfo, INodeExecutionData, @@ -18,12 +19,15 @@ import type { IWorkflowDataProxyData, ResourceMapperValue, } from 'n8n-workflow'; -import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; +import { + generateZodSchema, + jsonParse, + NodeConnectionType, + NodeOperationError, + traverseNodeParameters, +} from 'n8n-workflow'; import { z } from 'zod'; -import type { FromAIArgument } from './FromAIParser'; -import { AIParametersParser } from './FromAIParser'; - /** Main class for creating the Workflow tool Processes the node parameters and creates AI Agent tool capable of executing n8n workflows @@ -90,6 +94,9 @@ export class WorkflowToolService { const errorResponse = `There was an error: "${executionError.message}"`; void this.context.addOutputData(NodeConnectionType.AiTool, index, executionError); return errorResponse; + } finally { + // @ts-expect-error this accesses a private member on the actual implementation to fix https://linear.app/n8n/issue/ADO-3186/bug-workflowtool-v2-always-uses-first-row-of-input-data + this.context.runIndex++; } }; @@ -275,8 +282,7 @@ export class WorkflowToolService { description: string, func: (query: string | IDataObject, runManager?: CallbackManagerForToolRun) => Promise, ): Promise { - const fromAIParser = new AIParametersParser(this.context); - const collectedArguments = await this.extractFromAIParameters(fromAIParser); + const collectedArguments = await this.extractFromAIParameters(); // If there are no `fromAI` arguments, fallback to creating a simple tool if (collectedArguments.length === 0) { @@ -284,15 +290,13 @@ export class WorkflowToolService { } // Otherwise, prepare Zod schema and create a structured tool - const schema = this.createZodSchema(collectedArguments, fromAIParser); + const schema = this.createZodSchema(collectedArguments); return new DynamicStructuredTool({ schema, name, description, func }); } - private async extractFromAIParameters( - fromAIParser: AIParametersParser, - ): Promise { + private async extractFromAIParameters(): Promise { const collectedArguments: FromAIArgument[] = []; - fromAIParser.traverseNodeParameters(this.context.getNode().parameters, collectedArguments); + traverseNodeParameters(this.context.getNode().parameters, collectedArguments); const uniqueArgsMap = new Map(); for (const arg of collectedArguments) { @@ -302,9 +306,9 @@ export class WorkflowToolService { return Array.from(uniqueArgsMap.values()); } - private createZodSchema(args: FromAIArgument[], parser: AIParametersParser): z.ZodObject { + private createZodSchema(args: FromAIArgument[]): z.ZodObject { const schemaObj = args.reduce((acc: Record, placeholder) => { - acc[placeholder.key] = parser.generateZodSchema(placeholder); + acc[placeholder.key] = generateZodSchema(placeholder); return acc; }, {}); diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/versionDescription.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/versionDescription.ts index 469a7d6d4c..6d4275b449 100644 --- a/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/versionDescription.ts +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/versionDescription.ts @@ -107,7 +107,7 @@ export const versionDescription: INodeTypeDescription = { typeOptions: { loadOptionsDependsOn: ['workflowId.value'], resourceMapper: { - localResourceMapperMethod: 'loadWorkflowInputMappings', + localResourceMapperMethod: 'loadSubWorkflowInputs', valuesLabel: 'Workflow Inputs', mode: 'map', fieldWords: { diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts index 0323478ee8..d08bc2bab2 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts @@ -1,3 +1,4 @@ +import type { MemoryVectorStore } from 'langchain/vectorstores/memory'; import type { INodeProperties } from 'n8n-workflow'; import { createVectorStoreNode } from '../shared/createVectorStoreNode'; @@ -20,7 +21,7 @@ const insertFields: INodeProperties[] = [ }, ]; -export class VectorStoreInMemory extends createVectorStoreNode({ +export class VectorStoreInMemory extends createVectorStoreNode({ meta: { displayName: 'In-Memory Vector Store', name: 'vectorStoreInMemory', diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts index d9d5ee611a..7b2ab7664d 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.ts @@ -4,8 +4,8 @@ import { type PGVectorStoreArgs, } from '@langchain/community/vectorstores/pgvector'; import type { EmbeddingsInterface } from '@langchain/core/embeddings'; +import { configurePostgres } from 'n8n-nodes-base/dist/nodes/Postgres/transport'; import type { PostgresNodeCredentials } from 'n8n-nodes-base/dist/nodes/Postgres/v2/helpers/interfaces'; -import { configurePostgres } from 'n8n-nodes-base/dist/nodes/Postgres/v2/transport'; import type { INodeProperties } from 'n8n-workflow'; import type pg from 'pg'; @@ -213,7 +213,7 @@ class ExtendedPGVectorStore extends PGVectorStore { } } -export class VectorStorePGVector extends createVectorStoreNode({ +export class VectorStorePGVector extends createVectorStoreNode({ meta: { description: 'Work with your data in Postgresql with the PGVector extension', icon: 'file:postgres.svg', @@ -274,6 +274,7 @@ export class VectorStorePGVector extends createVectorStoreNode({ return await ExtendedPGVectorStore.initialize(embeddings, config); }, + async populateVectorStore(context, embeddings, documents, itemIndex) { // NOTE: if you are to create the HNSW index before use, you need to consider moving the distanceStrategy field to // shared fields, because you need that strategy when creating the index. @@ -307,6 +308,11 @@ export class VectorStorePGVector extends createVectorStoreNode({ metadataColumnName: 'metadata', }) as ColumnOptions; - await PGVectorStore.fromDocuments(documents, embeddings, config); + const vectorStore = await PGVectorStore.fromDocuments(documents, embeddings, config); + vectorStore.client?.release(); + }, + + releaseVectorStoreClient(vectorStore) { + vectorStore.client?.release(); }, }) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index 5a11acea24..61761a54ec 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -51,7 +51,7 @@ const insertFields: INodeProperties[] = [ }, ]; -export class VectorStorePinecone extends createVectorStoreNode({ +export class VectorStorePinecone extends createVectorStoreNode({ meta: { displayName: 'Pinecone Vector Store', name: 'vectorStorePinecone', diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts index 988f607ad7..e18cc4988e 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts @@ -79,7 +79,7 @@ const retrieveFields: INodeProperties[] = [ }, ]; -export class VectorStoreQdrant extends createVectorStoreNode({ +export class VectorStoreQdrant extends createVectorStoreNode({ meta: { displayName: 'Qdrant Vector Store', name: 'vectorStoreQdrant', diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts index a462ff8cf6..6ec3975ebd 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts @@ -41,7 +41,7 @@ const retrieveFields: INodeProperties[] = [ const updateFields: INodeProperties[] = [...insertFields]; -export class VectorStoreSupabase extends createVectorStoreNode({ +export class VectorStoreSupabase extends createVectorStoreNode({ meta: { description: 'Work with your data in Supabase Vector Store', icon: 'file:supabase.svg', diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts index 1372d54f6e..5c973002b2 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZep/VectorStoreZep.node.ts @@ -46,7 +46,7 @@ const retrieveFields: INodeProperties[] = [ }, ]; -export class VectorStoreZep extends createVectorStoreNode({ +export class VectorStoreZep extends createVectorStoreNode({ meta: { displayName: 'Zep Vector Store', name: 'vectorStoreZep', diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index 441126c985..ecf2e64a81 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -49,7 +49,7 @@ interface NodeMeta { operationModes?: NodeOperationMode[]; } -export interface VectorStoreNodeConstructorArgs { +export interface VectorStoreNodeConstructorArgs { meta: NodeMeta; methods?: { listSearch?: { @@ -77,7 +77,8 @@ export interface VectorStoreNodeConstructorArgs { filter: Record | undefined, embeddings: Embeddings, itemIndex: number, - ) => Promise; + ) => Promise; + releaseVectorStoreClient?: (vectorStore: T) => void; } function transformDescriptionForOperationMode( @@ -90,11 +91,15 @@ function transformDescriptionForOperationMode( })); } -function isUpdateSupported(args: VectorStoreNodeConstructorArgs): boolean { +function isUpdateSupported( + args: VectorStoreNodeConstructorArgs, +): boolean { return args.meta.operationModes?.includes('update') ?? false; } -function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] { +function getOperationModeOptions( + args: VectorStoreNodeConstructorArgs, +): INodePropertyOptions[] { const enabledOperationModes = args.meta.operationModes ?? DEFAULT_OPERATION_MODES; const allOptions = [ @@ -137,7 +142,9 @@ function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePro ); } -export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => +export const createVectorStoreNode = ( + args: VectorStoreNodeConstructorArgs, +) => class VectorStoreNodeType implements INodeType { description: INodeTypeDescription = { displayName: args.meta.displayName, @@ -334,38 +341,42 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => embeddings, itemIndex, ); - const prompt = this.getNodeParameter('prompt', itemIndex) as string; - const topK = this.getNodeParameter('topK', itemIndex, 4) as number; + try { + const prompt = this.getNodeParameter('prompt', itemIndex) as string; + const topK = this.getNodeParameter('topK', itemIndex, 4) as number; - const embeddedPrompt = await embeddings.embedQuery(prompt); - const docs = await vectorStore.similaritySearchVectorWithScore( - embeddedPrompt, - topK, - filter, - ); + const embeddedPrompt = await embeddings.embedQuery(prompt); + const docs = await vectorStore.similaritySearchVectorWithScore( + embeddedPrompt, + topK, + filter, + ); - const includeDocumentMetadata = this.getNodeParameter( - 'includeDocumentMetadata', - itemIndex, - true, - ) as boolean; + const includeDocumentMetadata = this.getNodeParameter( + 'includeDocumentMetadata', + itemIndex, + true, + ) as boolean; - const serializedDocs = docs.map(([doc, score]) => { - const document = { - pageContent: doc.pageContent, - ...(includeDocumentMetadata ? { metadata: doc.metadata } : {}), - }; + const serializedDocs = docs.map(([doc, score]) => { + const document = { + pageContent: doc.pageContent, + ...(includeDocumentMetadata ? { metadata: doc.metadata } : {}), + }; - return { - json: { document, score }, - pairedItem: { - item: itemIndex, - }, - }; - }); + return { + json: { document, score }, + pairedItem: { + item: itemIndex, + }, + }; + }); - resultData.push(...serializedDocs); - logAiEvent(this, 'ai-vector-store-searched', { query: prompt }); + resultData.push(...serializedDocs); + logAiEvent(this, 'ai-vector-store-searched', { query: prompt }); + } finally { + args.releaseVectorStoreClient?.(vectorStore); + } } return [resultData]; @@ -392,13 +403,9 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => ); resultData.push(...serializedDocuments); - try { - await args.populateVectorStore(this, embeddings, processedDocuments, itemIndex); + await args.populateVectorStore(this, embeddings, processedDocuments, itemIndex); - logAiEvent(this, 'ai-vector-store-populated'); - } catch (error) { - throw error; - } + logAiEvent(this, 'ai-vector-store-populated'); } return [resultData]; @@ -431,27 +438,27 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => itemIndex, ); - const { processedDocuments, serializedDocuments } = await processDocument( - loader, - itemData, - itemIndex, - ); - - if (processedDocuments?.length !== 1) { - throw new NodeOperationError(this.getNode(), 'Single document per item expected'); - } - - resultData.push(...serializedDocuments); - try { + const { processedDocuments, serializedDocuments } = await processDocument( + loader, + itemData, + itemIndex, + ); + + if (processedDocuments?.length !== 1) { + throw new NodeOperationError(this.getNode(), 'Single document per item expected'); + } + + resultData.push(...serializedDocuments); + // Use ids option to upsert instead of insert await vectorStore.addDocuments(processedDocuments, { ids: [documentId], }); logAiEvent(this, 'ai-vector-store-updated'); - } catch (error) { - throw error; + } finally { + args.releaseVectorStoreClient?.(vectorStore); } } @@ -476,6 +483,9 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const vectorStore = await args.getVectorStoreClient(this, filter, embeddings, itemIndex); return { response: logWrapper(vectorStore, this), + closeFunction: async () => { + args.releaseVectorStoreClient?.(vectorStore); + }, }; } @@ -499,23 +509,28 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => embeddings, itemIndex, ); - const embeddedPrompt = await embeddings.embedQuery(input); - const documents = await vectorStore.similaritySearchVectorWithScore( - embeddedPrompt, - topK, - filter, - ); - return documents - .map((document) => { - if (includeDocumentMetadata) { - return { type: 'text', text: JSON.stringify(document[0]) }; - } - return { - type: 'text', - text: JSON.stringify({ pageContent: document[0].pageContent }), - }; - }) - .filter((document) => !!document); + + try { + const embeddedPrompt = await embeddings.embedQuery(input); + const documents = await vectorStore.similaritySearchVectorWithScore( + embeddedPrompt, + topK, + filter, + ); + return documents + .map((document) => { + if (includeDocumentMetadata) { + return { type: 'text', text: JSON.stringify(document[0]) }; + } + return { + type: 'text', + text: JSON.stringify({ pageContent: document[0].pageContent }), + }; + }) + .filter((document) => !!document); + } finally { + args.releaseVectorStoreClient?.(vectorStore); + } }, }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/methods/listSearch.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/methods/listSearch.ts index dcf056618a..33c1b9ef2d 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/methods/listSearch.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/methods/listSearch.ts @@ -76,9 +76,16 @@ export async function modelSearch( this: ILoadOptionsFunctions, filter?: string, ): Promise { + const credentials = await this.getCredentials<{ url: string }>('openAiApi'); + const isCustomAPI = credentials.url && !credentials.url.includes('api.openai.com'); + return await getModelSearch( (model) => - model.id.startsWith('gpt-') || model.id.startsWith('ft:') || model.id.startsWith('o1'), + isCustomAPI || + model.id.startsWith('gpt-') || + model.id.startsWith('ft:') || + model.id.startsWith('o1') || + model.id.startsWith('o3'), )(this, filter); } diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index c7cc10aa77..519b9f6ad4 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -1,13 +1,13 @@ { "name": "@n8n/n8n-nodes-langchain", - "version": "1.74.0", + "version": "1.78.0", "description": "", "main": "index.js", "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm run watch", "typecheck": "tsc --noEmit", - "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && pnpm n8n-copy-icons && pnpm n8n-generate-metadata", + "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && pnpm n8n-copy-static-files && pnpm n8n-generate-metadata", "format": "biome format --write .", "format:check": "biome ci .", "lint": "eslint nodes credentials utils --quiet", @@ -25,12 +25,14 @@ "dist/credentials/AnthropicApi.credentials.js", "dist/credentials/AzureOpenAiApi.credentials.js", "dist/credentials/CohereApi.credentials.js", + "dist/credentials/DeepSeekApi.credentials.js", "dist/credentials/GooglePalmApi.credentials.js", "dist/credentials/GroqApi.credentials.js", "dist/credentials/HuggingFaceApi.credentials.js", "dist/credentials/MotorheadApi.credentials.js", "dist/credentials/MistralCloudApi.credentials.js", "dist/credentials/OllamaApi.credentials.js", + "dist/credentials/OpenRouterApi.credentials.js", "dist/credentials/PineconeApi.credentials.js", "dist/credentials/QdrantApi.credentials.js", "dist/credentials/SerpApi.credentials.js", @@ -64,11 +66,13 @@ "dist/nodes/llms/LMChatAnthropic/LmChatAnthropic.node.js", "dist/nodes/llms/LmChatAzureOpenAi/LmChatAzureOpenAi.node.js", "dist/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.js", + "dist/nodes/llms/LmChatDeepSeek/LmChatDeepSeek.node.js", "dist/nodes/llms/LmChatGoogleGemini/LmChatGoogleGemini.node.js", "dist/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.js", "dist/nodes/llms/LmChatGroq/LmChatGroq.node.js", "dist/nodes/llms/LmChatMistralCloud/LmChatMistralCloud.node.js", "dist/nodes/llms/LMChatOllama/LmChatOllama.node.js", + "dist/nodes/llms/LmChatOpenRouter/LmChatOpenRouter.node.js", "dist/nodes/llms/LMChatOpenAi/LmChatOpenAi.node.js", "dist/nodes/llms/LMOpenAi/LmOpenAi.node.js", "dist/nodes/llms/LMCohere/LmCohere.node.js", @@ -136,17 +140,17 @@ "@google-cloud/resource-manager": "5.3.0", "@google/generative-ai": "0.21.0", "@huggingface/inference": "2.8.0", - "@langchain/anthropic": "0.3.8", - "@langchain/aws": "0.1.2", - "@langchain/cohere": "0.3.1", - "@langchain/community": "0.3.15", + "@langchain/anthropic": "0.3.11", + "@langchain/aws": "0.1.3", + "@langchain/cohere": "0.3.2", + "@langchain/community": "0.3.24", "@langchain/core": "catalog:", - "@langchain/google-genai": "0.1.4", - "@langchain/google-vertexai": "0.1.3", - "@langchain/groq": "0.1.2", + "@langchain/google-genai": "0.1.6", + "@langchain/google-vertexai": "0.1.8", + "@langchain/groq": "0.1.3", "@langchain/mistralai": "0.2.0", - "@langchain/ollama": "0.1.2", - "@langchain/openai": "0.3.14", + "@langchain/ollama": "0.1.4", + "@langchain/openai": "0.3.17", "@langchain/pinecone": "0.1.3", "@langchain/qdrant": "0.1.1", "@langchain/redis": "0.1.0", @@ -168,13 +172,13 @@ "generate-schema": "2.6.0", "html-to-text": "9.0.5", "jsdom": "23.0.1", - "langchain": "0.3.6", + "langchain": "0.3.11", "lodash": "catalog:", "mammoth": "1.7.2", "mime-types": "2.1.35", "n8n-nodes-base": "workspace:*", "n8n-workflow": "workspace:*", - "openai": "4.73.1", + "openai": "4.78.1", "pdf-parse": "1.1.1", "pg": "8.12.0", "redis": "4.6.12", diff --git a/packages/@n8n/nodes-langchain/types/types.ts b/packages/@n8n/nodes-langchain/types/types.ts new file mode 100644 index 0000000000..75d1044132 --- /dev/null +++ b/packages/@n8n/nodes-langchain/types/types.ts @@ -0,0 +1 @@ +type OpenAICompatibleCredential = { apiKey: string; url: string }; diff --git a/packages/@n8n/storybook/package.json b/packages/@n8n/storybook/package.json index 6018fbd4b2..9506a7188e 100644 --- a/packages/@n8n/storybook/package.json +++ b/packages/@n8n/storybook/package.json @@ -3,19 +3,19 @@ "private": true, "version": "0.0.1", "devDependencies": { - "@chromatic-com/storybook": "^3.2.2", - "@storybook/addon-a11y": "^8.4.6", - "@storybook/addon-actions": "^8.4.6", - "@storybook/addon-docs": "^8.4.6", - "@storybook/addon-essentials": "^8.4.6", - "@storybook/addon-interactions": "^8.4.6", - "@storybook/addon-links": "^8.4.6", - "@storybook/addon-themes": "^8.4.6", - "@storybook/blocks": "^8.4.6", - "@storybook/test": "^8.4.6", - "@storybook/vue3": "^8.4.6", - "@storybook/vue3-vite": "^8.4.6", - "chromatic": "^11.20.0", - "storybook": "^8.4.6" + "@chromatic-com/storybook": "^3.2.4", + "@storybook/addon-a11y": "^8.5.0", + "@storybook/addon-actions": "^8.5.0", + "@storybook/addon-docs": "^8.5.0", + "@storybook/addon-essentials": "^8.5.0", + "@storybook/addon-interactions": "^8.5.0", + "@storybook/addon-links": "^8.5.0", + "@storybook/addon-themes": "^8.5.0", + "@storybook/blocks": "^8.5.0", + "@storybook/test": "^8.5.0", + "@storybook/vue3": "^8.5.0", + "@storybook/vue3-vite": "^8.5.0", + "chromatic": "^11.25.0", + "storybook": "^8.5.0" } } diff --git a/packages/@n8n/task-runner/package.json b/packages/@n8n/task-runner/package.json index c32ea2714e..b8782a252f 100644 --- a/packages/@n8n/task-runner/package.json +++ b/packages/@n8n/task-runner/package.json @@ -1,6 +1,6 @@ { "name": "@n8n/task-runner", - "version": "1.12.0", + "version": "1.16.0", "scripts": { "clean": "rimraf dist .turbo", "start": "node dist/start.js", @@ -40,13 +40,13 @@ "acorn": "8.14.0", "acorn-walk": "8.3.4", "lodash": "catalog:", + "luxon": "catalog:", "n8n-core": "workspace:*", "n8n-workflow": "workspace:*", "nanoid": "catalog:", "ws": "^8.18.0" }, "devDependencies": { - "@types/lodash": "catalog:", - "luxon": "catalog:" + "@types/lodash": "catalog:" } } diff --git a/packages/@n8n/task-runner/src/__tests__/task-runner-sentry.test.ts b/packages/@n8n/task-runner/src/__tests__/task-runner-sentry.test.ts new file mode 100644 index 0000000000..6fe14a484a --- /dev/null +++ b/packages/@n8n/task-runner/src/__tests__/task-runner-sentry.test.ts @@ -0,0 +1,178 @@ +import type { ErrorEvent } from '@sentry/types'; +import { mock } from 'jest-mock-extended'; +import type { ErrorReporter } from 'n8n-core'; + +import { TaskRunnerSentry } from '../task-runner-sentry'; + +describe('TaskRunnerSentry', () => { + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('filterOutUserCodeErrors', () => { + const sentry = new TaskRunnerSentry( + { + dsn: 'https://sentry.io/123', + n8nVersion: '1.0.0', + environment: 'local', + deploymentName: 'test', + }, + mock(), + ); + + it('should filter out user code errors', () => { + const event: ErrorEvent = { + type: undefined, + exception: { + values: [ + { + type: 'ReferenceError', + value: 'fetch is not defined', + stacktrace: { + frames: [ + { + filename: 'app:///dist/js-task-runner/js-task-runner.js', + module: 'js-task-runner:js-task-runner', + function: 'JsTaskRunner.executeTask', + }, + { + filename: 'app:///dist/js-task-runner/js-task-runner.js', + module: 'js-task-runner:js-task-runner', + function: 'JsTaskRunner.runForAllItems', + }, + { + filename: '', + module: '', + function: 'new Promise', + }, + { + filename: 'app:///dist/js-task-runner/js-task-runner.js', + module: 'js-task-runner:js-task-runner', + function: 'result', + }, + { + filename: 'node:vm', + module: 'node:vm', + function: 'runInContext', + }, + { + filename: 'node:vm', + module: 'node:vm', + function: 'Script.runInContext', + }, + { + filename: 'evalmachine.', + module: 'evalmachine.', + function: '?', + }, + { + filename: 'evalmachine.', + module: 'evalmachine.', + function: 'VmCodeWrapper', + }, + { + filename: '', + module: '', + function: 'new Promise', + }, + { + filename: 'evalmachine.', + module: 'evalmachine.', + }, + ], + }, + mechanism: { type: 'onunhandledrejection', handled: false }, + }, + ], + }, + event_id: '18bb78bb3d9d44c4acf3d774c2cfbfd8', + platform: 'node', + contexts: { + trace: { trace_id: '3c3614d33a6b47f09b85ec7d2710acea', span_id: 'ad00fdf6d6173aeb' }, + runtime: { name: 'node', version: 'v20.17.0' }, + }, + }; + + expect(sentry.filterOutUserCodeErrors(event)).toBe(true); + }); + }); + + describe('initIfEnabled', () => { + const mockErrorReporter = mock(); + + it('should not configure sentry if dsn is not set', async () => { + const sentry = new TaskRunnerSentry( + { + dsn: '', + n8nVersion: '1.0.0', + environment: 'local', + deploymentName: 'test', + }, + mockErrorReporter, + ); + + await sentry.initIfEnabled(); + + expect(mockErrorReporter.init).not.toHaveBeenCalled(); + }); + + it('should configure sentry if dsn is set', async () => { + const sentry = new TaskRunnerSentry( + { + dsn: 'https://sentry.io/123', + n8nVersion: '1.0.0', + environment: 'local', + deploymentName: 'test', + }, + mockErrorReporter, + ); + + await sentry.initIfEnabled(); + + expect(mockErrorReporter.init).toHaveBeenCalledWith({ + dsn: 'https://sentry.io/123', + beforeSendFilter: sentry.filterOutUserCodeErrors, + release: '1.0.0', + environment: 'local', + serverName: 'test', + serverType: 'task_runner', + }); + }); + }); + + describe('shutdown', () => { + const mockErrorReporter = mock(); + + it('should not shutdown sentry if dsn is not set', async () => { + const sentry = new TaskRunnerSentry( + { + dsn: '', + n8nVersion: '1.0.0', + environment: 'local', + deploymentName: 'test', + }, + mockErrorReporter, + ); + + await sentry.shutdown(); + + expect(mockErrorReporter.shutdown).not.toHaveBeenCalled(); + }); + + it('should shutdown sentry if dsn is set', async () => { + const sentry = new TaskRunnerSentry( + { + dsn: 'https://sentry.io/123', + n8nVersion: '1.0.0', + environment: 'local', + deploymentName: 'test', + }, + mockErrorReporter, + ); + + await sentry.shutdown(); + + expect(mockErrorReporter.shutdown).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/@n8n/task-runner/src/config/sentry-config.ts b/packages/@n8n/task-runner/src/config/sentry-config.ts index 691f64244f..ae24221a5e 100644 --- a/packages/@n8n/task-runner/src/config/sentry-config.ts +++ b/packages/@n8n/task-runner/src/config/sentry-config.ts @@ -2,9 +2,9 @@ import { Config, Env } from '@n8n/config'; @Config export class SentryConfig { - /** Sentry DSN */ + /** Sentry DSN (data source name) */ @Env('N8N_SENTRY_DSN') - sentryDsn: string = ''; + dsn: string = ''; //#region Metadata about the environment diff --git a/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts b/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts index 5ebd965e87..a666240098 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts @@ -1,4 +1,4 @@ -import { DateTime } from 'luxon'; +import { DateTime, Duration, Interval } from 'luxon'; import type { IBinaryData } from 'n8n-workflow'; import { setGlobalState, type CodeExecutionMode, type IDataObject } from 'n8n-workflow'; import fs from 'node:fs'; @@ -50,7 +50,7 @@ describe('JsTaskRunner', () => { ...jsRunnerOpts, }, sentryConfig: { - sentryDsn: '', + dsn: '', deploymentName: '', environment: '', n8nVersion: '', @@ -302,6 +302,7 @@ describe('JsTaskRunner', () => { ['typeof clearInterval', 'function'], ['typeof clearImmediate', 'function'], ], + eval: [['eval("1+2")', 3]], 'JS built-ins': [ ['typeof btoa', 'function'], ['typeof atob', 'function'], @@ -1341,4 +1342,98 @@ describe('JsTaskRunner', () => { task.cleanup(); }); }); + + describe('prototype pollution prevention', () => { + const checkPrototypeIntact = () => { + const obj: Record = {}; + expect(obj.maliciousKey).toBeUndefined(); + }; + + test('Object.setPrototypeOf should no-op for local object', async () => { + checkPrototypeIntact(); + + const outcome = await executeForAllItems({ + code: ` + const obj = {}; + Object.setPrototypeOf(obj, { maliciousKey: 'value' }); + return [{ json: { prototypeChanged: obj.maliciousKey !== undefined } }]; + `, + inputItems: [{ a: 1 }], + }); + + expect(outcome.result).toEqual([wrapIntoJson({ prototypeChanged: false })]); + checkPrototypeIntact(); + }); + + test('Reflect.setPrototypeOf should no-op for local object', async () => { + checkPrototypeIntact(); + + const outcome = await executeForAllItems({ + code: ` + const obj = {}; + Reflect.setPrototypeOf(obj, { maliciousKey: 'value' }); + return [{ json: { prototypeChanged: obj.maliciousKey !== undefined } }]; + `, + inputItems: [{ a: 1 }], + }); + + expect(outcome.result).toEqual([wrapIntoJson({ prototypeChanged: false })]); + checkPrototypeIntact(); + }); + + test('Object.setPrototypeOf should no-op for incoming object', async () => { + checkPrototypeIntact(); + + const outcome = await executeForAllItems({ + code: ` + const obj = $input.first(); + Object.setPrototypeOf(obj, { maliciousKey: 'value' }); + return [{ json: { prototypeChanged: obj.maliciousKey !== undefined } }]; + `, + inputItems: [{ a: 1 }], + }); + + expect(outcome.result).toEqual([wrapIntoJson({ prototypeChanged: false })]); + checkPrototypeIntact(); + }); + + test('Reflect.setPrototypeOf should no-op for incoming object', async () => { + checkPrototypeIntact(); + + const outcome = await executeForAllItems({ + code: ` + const obj = $input.first(); + Reflect.setPrototypeOf(obj, { maliciousKey: 'value' }); + return [{ json: { prototypeChanged: obj.maliciousKey !== undefined } }]; + `, + inputItems: [{ a: 1 }], + }); + + expect(outcome.result).toEqual([wrapIntoJson({ prototypeChanged: false })]); + checkPrototypeIntact(); + }); + + test('should freeze luxon prototypes', async () => { + const outcome = await executeForAllItems({ + code: ` + [DateTime, Interval, Duration] + .forEach(constructor => { + constructor.prototype.maliciousKey = 'value'; + }); + + return [] + `, + inputItems: [{ a: 1 }], + }); + + expect(outcome.result).toEqual([]); + + // @ts-expect-error Non-existing property + expect(DateTime.now().maliciousKey).toBeUndefined(); + // @ts-expect-error Non-existing property + expect(Interval.fromISO('P1Y2M10DT2H30M').maliciousKey).toBeUndefined(); + // @ts-expect-error Non-existing property + expect(Duration.fromObject({ hours: 1 }).maliciousKey).toBeUndefined(); + }); + }); }); diff --git a/packages/@n8n/task-runner/src/js-task-runner/__tests__/require-resolver.test.ts b/packages/@n8n/task-runner/src/js-task-runner/__tests__/require-resolver.test.ts new file mode 100644 index 0000000000..16e0f99e7c --- /dev/null +++ b/packages/@n8n/task-runner/src/js-task-runner/__tests__/require-resolver.test.ts @@ -0,0 +1,78 @@ +import { ApplicationError } from 'n8n-workflow'; + +import { ExecutionError } from '@/js-task-runner/errors/execution-error'; + +import { createRequireResolver, type RequireResolverOpts } from '../require-resolver'; + +describe('require resolver', () => { + let defaultOpts: RequireResolverOpts; + + beforeEach(() => { + defaultOpts = { + allowedBuiltInModules: new Set(['path', 'fs']), + allowedExternalModules: new Set(['lodash']), + }; + }); + + describe('built-in modules', () => { + it('should allow requiring whitelisted built-in modules', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('path')).not.toThrow(); + expect(() => resolver('fs')).not.toThrow(); + }); + + it('should throw when requiring non-whitelisted built-in modules', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('crypto')).toThrow(ExecutionError); + }); + + it('should allow all built-in modules when allowedBuiltInModules is "*"', () => { + const resolver = createRequireResolver({ + ...defaultOpts, + allowedBuiltInModules: '*', + }); + + expect(() => resolver('path')).not.toThrow(); + expect(() => resolver('crypto')).not.toThrow(); + expect(() => resolver('fs')).not.toThrow(); + }); + }); + + describe('external modules', () => { + it('should allow requiring whitelisted external modules', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('lodash')).not.toThrow(); + }); + + it('should throw when requiring non-whitelisted external modules', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('express')).toThrow( + new ExecutionError(new ApplicationError("Cannot find module 'express'")), + ); + }); + + it('should allow all external modules when allowedExternalModules is "*"', () => { + const resolver = createRequireResolver({ + ...defaultOpts, + allowedExternalModules: '*', + }); + + expect(() => resolver('lodash')).not.toThrow(); + expect(() => resolver('express')).not.toThrow(); + }); + }); + + describe('error handling', () => { + it('should wrap ApplicationError in ExecutionError', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('non-existent-module')).toThrow(ExecutionError); + }); + + it('should include the module name in the error message', () => { + const resolver = createRequireResolver(defaultOpts); + expect(() => resolver('non-existent-module')).toThrow( + "Cannot find module 'non-existent-module'", + ); + }); + }); +}); diff --git a/packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser.test.ts b/packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser.test.ts index dd866fc381..1f839b38f2 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser.test.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser.test.ts @@ -253,6 +253,7 @@ describe('BuiltInsParser', () => { '$node', '$self', '$parameter', + '$rawParameter', '$prevNode', '$runIndex', '$mode', diff --git a/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts b/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts index ab2cc3a304..25a8f8dfa4 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts @@ -1,6 +1,7 @@ import set from 'lodash/set'; +import { DateTime, Duration, Interval } from 'luxon'; import { getAdditionalKeys } from 'n8n-core'; -import { WorkflowDataProxy, Workflow, ObservableObject } from 'n8n-workflow'; +import { WorkflowDataProxy, Workflow, ObservableObject, Expression } from 'n8n-workflow'; import type { CodeExecutionMode, IWorkflowExecuteAdditionalData, @@ -19,7 +20,7 @@ import type { } from 'n8n-workflow'; import * as a from 'node:assert'; import { inspect } from 'node:util'; -import { runInNewContext, type Context } from 'node:vm'; +import { type Context, createContext, runInContext } from 'node:vm'; import type { MainConfig } from '@/config/main-config'; import { UnsupportedFunctionError } from '@/js-task-runner/errors/unsupported-function.error'; @@ -97,12 +98,55 @@ export class JsTaskRunner extends TaskRunner { const { jsRunnerConfig } = config; const parseModuleAllowList = (moduleList: string) => - moduleList === '*' ? null : new Set(moduleList.split(',').map((x) => x.trim())); + moduleList === '*' + ? '*' + : new Set( + moduleList + .split(',') + .map((x) => x.trim()) + .filter((x) => x !== ''), + ); + + const allowedBuiltInModules = parseModuleAllowList(jsRunnerConfig.allowedBuiltInModules ?? ''); + const allowedExternalModules = parseModuleAllowList( + jsRunnerConfig.allowedExternalModules ?? '', + ); this.requireResolver = createRequireResolver({ - allowedBuiltInModules: parseModuleAllowList(jsRunnerConfig.allowedBuiltInModules ?? ''), - allowedExternalModules: parseModuleAllowList(jsRunnerConfig.allowedExternalModules ?? ''), + allowedBuiltInModules, + allowedExternalModules, }); + + this.preventPrototypePollution(allowedExternalModules); + } + + private preventPrototypePollution(allowedExternalModules: Set | '*') { + if (allowedExternalModules instanceof Set) { + // This is a workaround to enable the allowed external libraries to mutate + // prototypes directly. For example momentjs overrides .toString() directly + // on the Moment.prototype, which doesn't work if Object.prototype has been + // frozen. This works as long as the overrides are done when the library is + // imported. + for (const module of allowedExternalModules) { + require(module); + } + } + + // Freeze globals, except for Jest + if (process.env.NODE_ENV !== 'test') { + Object.getOwnPropertyNames(globalThis) + // @ts-expect-error globalThis does not have string in index signature + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access + .map((name) => globalThis[name]) + .filter((value) => typeof value === 'function') + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access + .forEach((fn) => Object.freeze(fn.prototype)); + } + + // Freeze internal classes + [Workflow, Expression, WorkflowDataProxy, DateTime, Interval, Duration] + .map((constructor) => constructor.prototype) + .forEach(Object.freeze); } async executeTask( @@ -158,10 +202,8 @@ export class JsTaskRunner extends TaskRunner { private getNativeVariables() { return { - // Exposed Node.js globals in vm2 + // Exposed Node.js globals Buffer, - Function, - eval, setTimeout, setInterval, setImmediate, @@ -205,8 +247,11 @@ export class JsTaskRunner extends TaskRunner { signal.addEventListener('abort', abortHandler, { once: true }); - const taskResult = runInNewContext( - `globalThis.global = globalThis; module.exports = async function VmCodeWrapper() {${settings.code}\n}()`, + const preventPrototypeManipulation = + 'Object.getPrototypeOf = () => ({}); Reflect.getPrototypeOf = () => ({}); Object.setPrototypeOf = () => false; Reflect.setPrototypeOf = () => false;'; + + const taskResult = runInContext( + `globalThis.global = globalThis; ${preventPrototypeManipulation}; module.exports = async function VmCodeWrapper() {${settings.code}\n}()`, context, { timeout: this.taskTimeout * 1000 }, ) as Promise; @@ -268,7 +313,7 @@ export class JsTaskRunner extends TaskRunner { signal.addEventListener('abort', abortHandler); - const taskResult = runInNewContext( + const taskResult = runInContext( `module.exports = async function VmCodeWrapper() {${settings.code}\n}()`, context, { timeout: this.taskTimeout * 1000 }, @@ -463,14 +508,14 @@ export class JsTaskRunner extends TaskRunner { * @param dataProxy The data proxy object that provides access to built-ins * @param additionalProperties Additional properties to add to the context */ - private buildContext( + buildContext( taskId: string, workflow: Workflow, node: INode, dataProxy: IWorkflowDataProxyData, additionalProperties: Record = {}, ): Context { - const context: Context = { + return createContext({ [inspect.custom]: () => '[[ExecutionContext]]', require: this.requireResolver, module: {}, @@ -480,8 +525,6 @@ export class JsTaskRunner extends TaskRunner { ...dataProxy, ...this.buildRpcCallObject(taskId), ...additionalProperties, - }; - - return context; + }); } } diff --git a/packages/@n8n/task-runner/src/js-task-runner/require-resolver.ts b/packages/@n8n/task-runner/src/js-task-runner/require-resolver.ts index ffa00c0441..4facbd365c 100644 --- a/packages/@n8n/task-runner/src/js-task-runner/require-resolver.ts +++ b/packages/@n8n/task-runner/src/js-task-runner/require-resolver.ts @@ -6,15 +6,15 @@ import { ExecutionError } from './errors/execution-error'; export type RequireResolverOpts = { /** * List of built-in nodejs modules that are allowed to be required in the - * execution sandbox. `null` means all are allowed. + * execution sandbox. `"*"` means all are allowed. */ - allowedBuiltInModules: Set | null; + allowedBuiltInModules: Set | '*'; /** * List of external modules that are allowed to be required in the - * execution sandbox. `null` means all are allowed. + * execution sandbox. `"*"` means all are allowed. */ - allowedExternalModules: Set | null; + allowedExternalModules: Set | '*'; }; export type RequireResolver = (request: string) => unknown; @@ -24,8 +24,8 @@ export function createRequireResolver({ allowedExternalModules, }: RequireResolverOpts) { return (request: string) => { - const checkIsAllowed = (allowList: Set | null, moduleName: string) => { - return allowList ? allowList.has(moduleName) : true; + const checkIsAllowed = (allowList: Set | '*', moduleName: string) => { + return allowList === '*' || allowList.has(moduleName); }; const isAllowed = isBuiltin(request) diff --git a/packages/@n8n/task-runner/src/start.ts b/packages/@n8n/task-runner/src/start.ts index 536b550d17..35b928e9a8 100644 --- a/packages/@n8n/task-runner/src/start.ts +++ b/packages/@n8n/task-runner/src/start.ts @@ -1,16 +1,16 @@ import './polyfills'; import { Container } from '@n8n/di'; -import type { ErrorReporter } from 'n8n-core'; import { ensureError, setGlobalState } from 'n8n-workflow'; import { MainConfig } from './config/main-config'; import type { HealthCheckServer } from './health-check-server'; import { JsTaskRunner } from './js-task-runner/js-task-runner'; +import { TaskRunnerSentry } from './task-runner-sentry'; let healthCheckServer: HealthCheckServer | undefined; let runner: JsTaskRunner | undefined; let isShuttingDown = false; -let errorReporter: ErrorReporter | undefined; +let sentry: TaskRunnerSentry | undefined; function createSignalHandler(signal: string, timeoutInS = 10) { return async function onSignal() { @@ -33,9 +33,9 @@ function createSignalHandler(signal: string, timeoutInS = 10) { void healthCheckServer?.stop(); } - if (errorReporter) { - await errorReporter.shutdown(); - errorReporter = undefined; + if (sentry) { + await sentry.shutdown(); + sentry = undefined; } } catch (e) { const error = ensureError(e); @@ -54,11 +54,8 @@ void (async function start() { defaultTimezone: config.baseRunnerConfig.timezone, }); - if (config.sentryConfig.sentryDsn) { - const { ErrorReporter } = await import('n8n-core'); - errorReporter = Container.get(ErrorReporter); - await errorReporter.init('task_runner', config.sentryConfig.sentryDsn); - } + sentry = Container.get(TaskRunnerSentry); + await sentry.initIfEnabled(); runner = new JsTaskRunner(config); runner.on('runner:reached-idle-timeout', () => { diff --git a/packages/@n8n/task-runner/src/task-runner-sentry.ts b/packages/@n8n/task-runner/src/task-runner-sentry.ts new file mode 100644 index 0000000000..5443a98d5c --- /dev/null +++ b/packages/@n8n/task-runner/src/task-runner-sentry.ts @@ -0,0 +1,62 @@ +import { Service } from '@n8n/di'; +import type { ErrorEvent, Exception } from '@sentry/types'; +import { ErrorReporter } from 'n8n-core'; + +import { SentryConfig } from './config/sentry-config'; + +/** + * Sentry service for the task runner. + */ +@Service() +export class TaskRunnerSentry { + constructor( + private readonly config: SentryConfig, + private readonly errorReporter: ErrorReporter, + ) {} + + async initIfEnabled() { + const { dsn, n8nVersion, environment, deploymentName } = this.config; + + if (!dsn) return; + + await this.errorReporter.init({ + serverType: 'task_runner', + dsn, + release: n8nVersion, + environment, + serverName: deploymentName, + beforeSendFilter: this.filterOutUserCodeErrors, + }); + } + + async shutdown() { + if (!this.config.dsn) return; + + await this.errorReporter.shutdown(); + } + + /** + * Filter out errors originating from user provided code. + * It is possible for users to create code that causes unhandledrejections + * that end up in the sentry error reporting. + */ + filterOutUserCodeErrors = (event: ErrorEvent) => { + const error = event?.exception?.values?.[0]; + + return error ? this.isUserCodeError(error) : false; + }; + + /** + * Check if the error is originating from user provided code. + * It is possible for users to create code that causes unhandledrejections + * that end up in the sentry error reporting. + */ + private isUserCodeError(error: Exception) { + const frames = error.stacktrace?.frames; + if (!frames) return false; + + return frames.some( + (frame) => frame.filename === 'node:vm' && frame.function === 'runInContext', + ); + } +} diff --git a/packages/@n8n_io/eslint-config/base.js b/packages/@n8n_io/eslint-config/base.js index 0864a20a7b..dc4ad37025 100644 --- a/packages/@n8n_io/eslint-config/base.js +++ b/packages/@n8n_io/eslint-config/base.js @@ -369,6 +369,8 @@ const config = (module.exports = { 'n8n-local-rules/no-unused-param-in-catch-clause': 'error', + 'n8n-local-rules/no-useless-catch-throw': 'error', + 'n8n-local-rules/no-plain-errors': 'error', // ****************************************************************** diff --git a/packages/@n8n_io/eslint-config/local-rules.js b/packages/@n8n_io/eslint-config/local-rules.js index 92b0f6669e..d32d72f89c 100644 --- a/packages/@n8n_io/eslint-config/local-rules.js +++ b/packages/@n8n_io/eslint-config/local-rules.js @@ -172,6 +172,49 @@ module.exports = { }, }, + 'no-useless-catch-throw': { + meta: { + type: 'problem', + docs: { + description: 'Disallow `try-catch` blocks where the `catch` only contains a `throw error`.', + recommended: 'error', + }, + messages: { + noUselessCatchThrow: 'Remove useless `catch` block.', + }, + fixable: 'code', + }, + create(context) { + return { + CatchClause(node) { + if ( + node.body.body.length === 1 && + node.body.body[0].type === 'ThrowStatement' && + node.body.body[0].argument.type === 'Identifier' && + node.body.body[0].argument.name === node.param.name + ) { + context.report({ + node, + messageId: 'noUselessCatchThrow', + fix(fixer) { + const tryStatement = node.parent; + const tryBlock = tryStatement.block; + const sourceCode = context.getSourceCode(); + const tryBlockText = sourceCode.getText(tryBlock); + const tryBlockTextWithoutBraces = tryBlockText.slice(1, -1).trim(); + const indentedTryBlockText = tryBlockTextWithoutBraces + .split('\n') + .map((line) => line.replace(/\t/, '')) + .join('\n'); + return fixer.replaceText(tryStatement, indentedTryBlockText); + }, + }); + } + }, + }; + }, + }, + 'no-skipped-tests': { meta: { type: 'problem', diff --git a/packages/@n8n_io/eslint-config/local-rules.test.js b/packages/@n8n_io/eslint-config/local-rules.test.js index 86589ac67f..ca32c37a40 100644 --- a/packages/@n8n_io/eslint-config/local-rules.test.js +++ b/packages/@n8n_io/eslint-config/local-rules.test.js @@ -51,3 +51,33 @@ ruleTester.run('no-json-parse-json-stringify', rules['no-json-parse-json-stringi }, ], }); + +ruleTester.run('no-useless-catch-throw', rules['no-useless-catch-throw'], { + valid: [ + { + code: 'try { foo(); } catch (e) { console.error(e); }', + }, + { + code: 'try { foo(); } catch (e) { throw new Error("Custom error"); }', + }, + ], + invalid: [ + { + code: ` +try { + // Some comment + if (foo) { + bar(); + } +} catch (e) { + throw e; +}`, + errors: [{ messageId: 'noUselessCatchThrow' }], + output: ` +// Some comment +if (foo) { + bar(); +}`, + }, + ], +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index 52bbdd3e6f..7c4f1b9a3c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "1.74.1", + "version": "1.78.0", "description": "n8n Workflow Automation Tool", "main": "dist/index", "types": "dist/index.d.ts", @@ -96,7 +96,7 @@ "@n8n/task-runner": "workspace:*", "@n8n/typeorm": "0.3.20-12", "@n8n_io/ai-assistant-sdk": "1.13.0", - "@n8n_io/license-sdk": "2.13.1", + "@n8n_io/license-sdk": "2.15.1", "@oclif/core": "4.0.7", "@rudderstack/rudder-sdk-node": "2.0.9", "@sentry/node": "catalog:", @@ -112,7 +112,6 @@ "convict": "6.2.4", "cookie-parser": "1.4.7", "csrf": "3.1.0", - "curlconverter": "3.21.0", "dotenv": "8.6.0", "express": "4.21.1", "express-async-errors": "3.1.1", diff --git a/packages/cli/src/__tests__/active-executions.test.ts b/packages/cli/src/__tests__/active-executions.test.ts index 5432bf5c4a..0ae648a098 100644 --- a/packages/cli/src/__tests__/active-executions.test.ts +++ b/packages/cli/src/__tests__/active-executions.test.ts @@ -1,28 +1,29 @@ -import { mock } from 'jest-mock-extended'; +import { captor, mock } from 'jest-mock-extended'; import type { + IDeferredPromise, IExecuteResponsePromiseData, IRun, IWorkflowExecutionDataProcess, } from 'n8n-workflow'; -import { createDeferredPromise } from 'n8n-workflow'; +import { ExecutionCancelledError, randomInt, sleep } from 'n8n-workflow'; import PCancelable from 'p-cancelable'; import { v4 as uuid } from 'uuid'; import { ActiveExecutions } from '@/active-executions'; import { ConcurrencyControlService } from '@/concurrency/concurrency-control.service'; +import config from '@/config'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { mockInstance } from '@test/mocking'; +jest.mock('n8n-workflow', () => ({ + ...jest.requireActual('n8n-workflow'), + sleep: jest.fn(), +})); + const FAKE_EXECUTION_ID = '15'; const FAKE_SECOND_EXECUTION_ID = '20'; -const updateExistingExecution = jest.fn(); -const createNewExecution = jest.fn(async () => FAKE_EXECUTION_ID); - -const executionRepository = mock({ - updateExistingExecution, - createNewExecution, -}); +const executionRepository = mock(); const concurrencyControl = mockInstance(ConcurrencyControlService, { // @ts-expect-error Private property @@ -31,125 +32,22 @@ const concurrencyControl = mockInstance(ConcurrencyControlService, { describe('ActiveExecutions', () => { let activeExecutions: ActiveExecutions; + let responsePromise: IDeferredPromise; + let workflowExecution: PCancelable; + let postExecutePromise: Promise; - beforeEach(() => { - activeExecutions = new ActiveExecutions(mock(), executionRepository, concurrencyControl); - }); + const fullRunData: IRun = { + data: { + resultData: { + runData: {}, + }, + }, + mode: 'manual', + startedAt: new Date(), + status: 'new', + }; - afterEach(() => { - jest.clearAllMocks(); - }); - - test('Should initialize activeExecutions with empty list', () => { - expect(activeExecutions.getActiveExecutions().length).toBe(0); - }); - - test('Should add execution to active execution list', async () => { - const newExecution = mockExecutionData(); - const executionId = await activeExecutions.add(newExecution); - - expect(executionId).toBe(FAKE_EXECUTION_ID); - expect(activeExecutions.getActiveExecutions().length).toBe(1); - expect(createNewExecution).toHaveBeenCalledTimes(1); - expect(updateExistingExecution).toHaveBeenCalledTimes(0); - }); - - test('Should update execution if add is called with execution ID', async () => { - const newExecution = mockExecutionData(); - const executionId = await activeExecutions.add(newExecution, FAKE_SECOND_EXECUTION_ID); - - expect(executionId).toBe(FAKE_SECOND_EXECUTION_ID); - expect(activeExecutions.getActiveExecutions().length).toBe(1); - expect(createNewExecution).toHaveBeenCalledTimes(0); - expect(updateExistingExecution).toHaveBeenCalledTimes(1); - }); - - test('Should fail attaching execution to invalid executionId', async () => { - const deferredPromise = mockCancelablePromise(); - - expect(() => { - activeExecutions.attachWorkflowExecution(FAKE_EXECUTION_ID, deferredPromise); - }).toThrow(); - }); - - test('Should successfully attach execution to valid executionId', async () => { - const newExecution = mockExecutionData(); - await activeExecutions.add(newExecution, FAKE_EXECUTION_ID); - const deferredPromise = mockCancelablePromise(); - - expect(() => - activeExecutions.attachWorkflowExecution(FAKE_EXECUTION_ID, deferredPromise), - ).not.toThrow(); - }); - - test('Should attach and resolve response promise to existing execution', async () => { - const newExecution = mockExecutionData(); - await activeExecutions.add(newExecution, FAKE_EXECUTION_ID); - const deferredPromise = mockDeferredPromise(); - activeExecutions.attachResponsePromise(FAKE_EXECUTION_ID, deferredPromise); - const fakeResponse = { data: { resultData: { runData: {} } } }; - activeExecutions.resolveResponsePromise(FAKE_EXECUTION_ID, fakeResponse); - - await expect(deferredPromise.promise).resolves.toEqual(fakeResponse); - }); - - test('Should remove an existing execution', async () => { - // ARRANGE - const newExecution = mockExecutionData(); - const executionId = await activeExecutions.add(newExecution); - - // ACT - activeExecutions.finalizeExecution(executionId); - - // Wait until the next tick to ensure that the post-execution promise has settled - await new Promise(setImmediate); - - // ASSERT - expect(activeExecutions.getActiveExecutions().length).toBe(0); - }); - - test('Should not try to resolve a post-execute promise for an inactive execution', async () => { - // @ts-expect-error Private method - const getExecutionSpy = jest.spyOn(activeExecutions, 'getExecution'); - - activeExecutions.finalizeExecution('inactive-execution-id', mockFullRunData()); - - expect(getExecutionSpy).not.toHaveBeenCalled(); - }); - - test('Should resolve post execute promise on removal', async () => { - const newExecution = mockExecutionData(); - const executionId = await activeExecutions.add(newExecution); - const postExecutePromise = activeExecutions.getPostExecutePromise(executionId); - // Force the above to be executed since we cannot await it - await new Promise((res) => { - setTimeout(res, 100); - }); - const fakeOutput = mockFullRunData(); - activeExecutions.finalizeExecution(executionId, fakeOutput); - - await expect(postExecutePromise).resolves.toEqual(fakeOutput); - }); - - test('Should throw error when trying to create a promise with invalid execution', async () => { - await expect(activeExecutions.getPostExecutePromise(FAKE_EXECUTION_ID)).rejects.toThrow(); - }); - - test('Should call function to cancel execution when asked to stop', async () => { - const newExecution = mockExecutionData(); - const executionId = await activeExecutions.add(newExecution); - const cancelExecution = jest.fn(); - const cancellablePromise = mockCancelablePromise(); - cancellablePromise.cancel = cancelExecution; - activeExecutions.attachWorkflowExecution(executionId, cancellablePromise); - activeExecutions.stopExecution(executionId); - - expect(cancelExecution).toHaveBeenCalledTimes(1); - }); -}); - -function mockExecutionData(): IWorkflowExecutionDataProcess { - return { + const executionData: IWorkflowExecutionDataProcess = { executionMode: 'manual', workflowData: { id: '123', @@ -162,22 +60,235 @@ function mockExecutionData(): IWorkflowExecutionDataProcess { }, userId: uuid(), }; -} -function mockFullRunData(): IRun { - return { - data: { - resultData: { - runData: {}, - }, - }, - mode: 'manual', - startedAt: new Date(), - status: 'new', - }; -} + beforeEach(() => { + activeExecutions = new ActiveExecutions(mock(), executionRepository, concurrencyControl); -// eslint-disable-next-line @typescript-eslint/promise-function-async -const mockCancelablePromise = () => new PCancelable((resolve) => resolve()); + executionRepository.createNewExecution.mockResolvedValue(FAKE_EXECUTION_ID); -const mockDeferredPromise = () => createDeferredPromise(); + workflowExecution = new PCancelable((resolve) => resolve()); + workflowExecution.cancel = jest.fn(); + responsePromise = mock>(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Should initialize activeExecutions with empty list', () => { + expect(activeExecutions.getActiveExecutions()).toHaveLength(0); + }); + + test('Should add execution to active execution list', async () => { + const executionId = await activeExecutions.add(executionData); + + expect(executionId).toBe(FAKE_EXECUTION_ID); + expect(activeExecutions.getActiveExecutions()).toHaveLength(1); + expect(executionRepository.createNewExecution).toHaveBeenCalledTimes(1); + expect(executionRepository.updateExistingExecution).toHaveBeenCalledTimes(0); + }); + + test('Should update execution if add is called with execution ID', async () => { + const executionId = await activeExecutions.add(executionData, FAKE_SECOND_EXECUTION_ID); + + expect(executionId).toBe(FAKE_SECOND_EXECUTION_ID); + expect(activeExecutions.getActiveExecutions()).toHaveLength(1); + expect(executionRepository.createNewExecution).toHaveBeenCalledTimes(0); + expect(executionRepository.updateExistingExecution).toHaveBeenCalledTimes(1); + }); + + describe('attachWorkflowExecution', () => { + test('Should fail attaching execution to invalid executionId', async () => { + expect(() => { + activeExecutions.attachWorkflowExecution(FAKE_EXECUTION_ID, workflowExecution); + }).toThrow(); + }); + + test('Should successfully attach execution to valid executionId', async () => { + await activeExecutions.add(executionData, FAKE_EXECUTION_ID); + + expect(() => + activeExecutions.attachWorkflowExecution(FAKE_EXECUTION_ID, workflowExecution), + ).not.toThrow(); + }); + }); + + test('Should attach and resolve response promise to existing execution', async () => { + await activeExecutions.add(executionData, FAKE_EXECUTION_ID); + activeExecutions.attachResponsePromise(FAKE_EXECUTION_ID, responsePromise); + const fakeResponse = { data: { resultData: { runData: {} } } }; + activeExecutions.resolveResponsePromise(FAKE_EXECUTION_ID, fakeResponse); + + expect(responsePromise.resolve).toHaveBeenCalledWith(fakeResponse); + }); + + test('Should copy over startedAt and responsePromise when resuming a waiting execution', async () => { + const executionId = await activeExecutions.add(executionData); + activeExecutions.setStatus(executionId, 'waiting'); + activeExecutions.attachResponsePromise(executionId, responsePromise); + + const waitingExecution = activeExecutions.getExecutionOrFail(executionId); + expect(waitingExecution.responsePromise).toBeDefined(); + + // Resume the execution + await activeExecutions.add(executionData, executionId); + + const resumedExecution = activeExecutions.getExecutionOrFail(executionId); + expect(resumedExecution.startedAt).toBe(waitingExecution.startedAt); + expect(resumedExecution.responsePromise).toBe(responsePromise); + }); + + describe('finalizeExecution', () => { + test('Should not remove a waiting execution', async () => { + const executionId = await activeExecutions.add(executionData); + activeExecutions.setStatus(executionId, 'waiting'); + activeExecutions.finalizeExecution(executionId); + + // Wait until the next tick to ensure that the post-execution promise has settled + await new Promise(setImmediate); + + // Execution should still be in activeExecutions + expect(activeExecutions.getActiveExecutions()).toHaveLength(1); + expect(activeExecutions.getStatus(executionId)).toBe('waiting'); + }); + + test('Should remove an existing execution', async () => { + const executionId = await activeExecutions.add(executionData); + + activeExecutions.finalizeExecution(executionId); + + await new Promise(setImmediate); + expect(activeExecutions.getActiveExecutions()).toHaveLength(0); + }); + + test('Should not try to resolve a post-execute promise for an inactive execution', async () => { + const getExecutionSpy = jest.spyOn(activeExecutions, 'getExecutionOrFail'); + + activeExecutions.finalizeExecution('inactive-execution-id', fullRunData); + + expect(getExecutionSpy).not.toHaveBeenCalled(); + }); + + test('Should resolve post execute promise on removal', async () => { + const executionId = await activeExecutions.add(executionData); + const postExecutePromise = activeExecutions.getPostExecutePromise(executionId); + + await new Promise(setImmediate); + activeExecutions.finalizeExecution(executionId, fullRunData); + + await expect(postExecutePromise).resolves.toEqual(fullRunData); + }); + }); + + describe('getPostExecutePromise', () => { + test('Should throw error when trying to create a promise with invalid execution', async () => { + await expect(activeExecutions.getPostExecutePromise(FAKE_EXECUTION_ID)).rejects.toThrow(); + }); + }); + + describe('stopExecution', () => { + let executionId: string; + + beforeEach(async () => { + executionId = await activeExecutions.add(executionData); + postExecutePromise = activeExecutions.getPostExecutePromise(executionId); + + activeExecutions.attachWorkflowExecution(executionId, workflowExecution); + activeExecutions.attachResponsePromise(executionId, responsePromise); + }); + + test('Should cancel ongoing executions', async () => { + activeExecutions.stopExecution(executionId); + + expect(responsePromise.reject).toHaveBeenCalledWith(expect.any(ExecutionCancelledError)); + expect(workflowExecution.cancel).toHaveBeenCalledTimes(1); + await expect(postExecutePromise).rejects.toThrow(ExecutionCancelledError); + }); + + test('Should cancel waiting executions', async () => { + activeExecutions.setStatus(executionId, 'waiting'); + activeExecutions.stopExecution(executionId); + + expect(responsePromise.reject).toHaveBeenCalledWith(expect.any(ExecutionCancelledError)); + expect(workflowExecution.cancel).not.toHaveBeenCalled(); + }); + }); + + describe('shutdown', () => { + let newExecutionId1: string, newExecutionId2: string; + let waitingExecutionId1: string, waitingExecutionId2: string; + + beforeEach(async () => { + config.set('executions.mode', 'regular'); + + executionRepository.createNewExecution.mockImplementation(async () => + randomInt(1000, 2000).toString(), + ); + + (sleep as jest.Mock).mockImplementation(() => { + // @ts-expect-error private property + activeExecutions.activeExecutions = {}; + }); + + newExecutionId1 = await activeExecutions.add(executionData); + activeExecutions.setStatus(newExecutionId1, 'new'); + activeExecutions.attachResponsePromise(newExecutionId1, responsePromise); + + newExecutionId2 = await activeExecutions.add(executionData); + activeExecutions.setStatus(newExecutionId2, 'new'); + + waitingExecutionId1 = await activeExecutions.add(executionData); + activeExecutions.setStatus(waitingExecutionId1, 'waiting'); + activeExecutions.attachResponsePromise(waitingExecutionId1, responsePromise); + + waitingExecutionId2 = await activeExecutions.add(executionData); + activeExecutions.setStatus(waitingExecutionId2, 'waiting'); + }); + + test('Should cancel only executions with response-promises by default', async () => { + const stopExecutionSpy = jest.spyOn(activeExecutions, 'stopExecution'); + + expect(activeExecutions.getActiveExecutions()).toHaveLength(4); + + await activeExecutions.shutdown(); + + expect(concurrencyControl.disable).toHaveBeenCalled(); + + const removeAllCaptor = captor(); + expect(concurrencyControl.removeAll).toHaveBeenCalledWith(removeAllCaptor); + expect(removeAllCaptor.value.sort()).toEqual([newExecutionId1, waitingExecutionId1].sort()); + + expect(stopExecutionSpy).toHaveBeenCalledTimes(2); + expect(stopExecutionSpy).toHaveBeenCalledWith(newExecutionId1); + expect(stopExecutionSpy).toHaveBeenCalledWith(waitingExecutionId1); + expect(stopExecutionSpy).not.toHaveBeenCalledWith(newExecutionId2); + expect(stopExecutionSpy).not.toHaveBeenCalledWith(waitingExecutionId2); + + await new Promise(setImmediate); + // the other two executions aren't cancelled, but still removed from memory + expect(activeExecutions.getActiveExecutions()).toHaveLength(0); + }); + + test('Should cancel all executions when cancelAll is true', async () => { + const stopExecutionSpy = jest.spyOn(activeExecutions, 'stopExecution'); + + expect(activeExecutions.getActiveExecutions()).toHaveLength(4); + + await activeExecutions.shutdown(true); + + expect(concurrencyControl.disable).toHaveBeenCalled(); + + const removeAllCaptor = captor(); + expect(concurrencyControl.removeAll).toHaveBeenCalledWith(removeAllCaptor); + expect(removeAllCaptor.value.sort()).toEqual( + [newExecutionId1, newExecutionId2, waitingExecutionId1, waitingExecutionId2].sort(), + ); + + expect(stopExecutionSpy).toHaveBeenCalledTimes(4); + expect(stopExecutionSpy).toHaveBeenCalledWith(newExecutionId1); + expect(stopExecutionSpy).toHaveBeenCalledWith(waitingExecutionId1); + expect(stopExecutionSpy).toHaveBeenCalledWith(newExecutionId2); + expect(stopExecutionSpy).toHaveBeenCalledWith(waitingExecutionId2); + }); + }); +}); diff --git a/packages/cli/src/__tests__/external-hooks.test.ts b/packages/cli/src/__tests__/external-hooks.test.ts new file mode 100644 index 0000000000..5e4aa6ee89 --- /dev/null +++ b/packages/cli/src/__tests__/external-hooks.test.ts @@ -0,0 +1,125 @@ +import type { GlobalConfig } from '@n8n/config'; +import { mock } from 'jest-mock-extended'; +import type { ErrorReporter, Logger } from 'n8n-core'; +import type { IWorkflowBase } from 'n8n-workflow'; +import { ApplicationError } from 'n8n-workflow'; + +import type { CredentialsRepository } from '@/databases/repositories/credentials.repository'; +import type { SettingsRepository } from '@/databases/repositories/settings.repository'; +import type { UserRepository } from '@/databases/repositories/user.repository'; +import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import { ExternalHooks } from '@/external-hooks'; + +describe('ExternalHooks', () => { + const logger = mock(); + const errorReporter = mock(); + const globalConfig = mock(); + const userRepository = mock(); + const settingsRepository = mock(); + const credentialsRepository = mock(); + const workflowRepository = mock(); + + const workflowData = mock({ id: '123', name: 'Test Workflow' }); + const hookFn = jest.fn(); + + let externalHooks: ExternalHooks; + + beforeEach(() => { + jest.resetAllMocks(); + globalConfig.externalHooks.files = []; + externalHooks = new ExternalHooks( + logger, + errorReporter, + globalConfig, + userRepository, + settingsRepository, + credentialsRepository, + workflowRepository, + ); + }); + + describe('init()', () => { + it('should not load hooks if no external hook files are configured', async () => { + // @ts-expect-error private method + const loadHooksSpy = jest.spyOn(externalHooks, 'loadHooks'); + await externalHooks.init(); + expect(loadHooksSpy).not.toHaveBeenCalled(); + }); + + it('should throw an error if hook file cannot be loaded', async () => { + globalConfig.externalHooks.files = ['/path/to/non-existent-hook.js']; + + jest.mock( + '/path/to/non-existent-hook.js', + () => { + throw new Error('File not found'); + }, + { virtual: true }, + ); + + await expect(externalHooks.init()).rejects.toThrow(ApplicationError); + }); + + it('should successfully load hooks from valid hook file', async () => { + const mockHookFile = { + workflow: { + create: [hookFn], + }, + }; + + globalConfig.externalHooks.files = ['/path/to/valid-hook.js']; + jest.mock('/path/to/valid-hook.js', () => mockHookFile, { virtual: true }); + + await externalHooks.init(); + + // eslint-disable-next-line @typescript-eslint/dot-notation + expect(externalHooks['registered']['workflow.create']).toHaveLength(1); + + await externalHooks.run('workflow.create', [workflowData]); + + expect(hookFn).toHaveBeenCalledTimes(1); + expect(hookFn).toHaveBeenCalledWith(workflowData); + }); + }); + + describe('run()', () => { + it('should not throw if no hooks are registered', async () => { + await externalHooks.run('n8n.stop'); + }); + + it('should execute registered hooks', async () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + externalHooks['registered']['workflow.create'] = [hookFn]; + + await externalHooks.run('workflow.create', [workflowData]); + + expect(hookFn).toHaveBeenCalledTimes(1); + + const hookInvocationContext = hookFn.mock.instances[0]; + expect(hookInvocationContext).toHaveProperty('dbCollections'); + expect(hookInvocationContext.dbCollections).toEqual({ + User: userRepository, + Settings: settingsRepository, + Credentials: credentialsRepository, + Workflow: workflowRepository, + }); + }); + + it('should report error if hook execution fails', async () => { + hookFn.mockRejectedValueOnce(new Error('Hook failed')); + // eslint-disable-next-line @typescript-eslint/dot-notation + externalHooks['registered']['workflow.create'] = [hookFn]; + + await expect(externalHooks.run('workflow.create', [workflowData])).rejects.toThrow( + ApplicationError, + ); + + expect(errorReporter.error).toHaveBeenCalledWith(expect.any(ApplicationError), { + level: 'fatal', + }); + expect(logger.error).toHaveBeenCalledWith( + 'There was a problem running hook "workflow.create"', + ); + }); + }); +}); diff --git a/packages/cli/src/__tests__/license.test.ts b/packages/cli/src/__tests__/license.test.ts index aa0aba1d53..fb82f312ee 100644 --- a/packages/cli/src/__tests__/license.test.ts +++ b/packages/cli/src/__tests__/license.test.ts @@ -3,7 +3,6 @@ import { LicenseManager } from '@n8n_io/license-sdk'; import { mock } from 'jest-mock-extended'; import type { InstanceSettings } from 'n8n-core'; -import config from '@/config'; import { N8N_VERSION } from '@/constants'; import { License } from '@/license'; import { mockLogger } from '@test/mocking'; @@ -17,6 +16,12 @@ const MOCK_ACTIVATION_KEY = 'activation-key'; const MOCK_FEATURE_FLAG = 'feat:sharing'; const MOCK_MAIN_PLAN_ID = '1b765dc4-d39d-4ffe-9885-c56dd67c4b26'; +function makeDateWithHourOffset(offsetInHours: number): Date { + const date = new Date(); + date.setHours(date.getHours() + offsetInHours); + return date; +} + const licenseConfig: GlobalConfig['license'] = { serverUrl: MOCK_SERVER_URL, autoRenewalEnabled: true, @@ -31,6 +36,7 @@ describe('License', () => { const instanceSettings = mock({ instanceId: MOCK_INSTANCE_ID, instanceType: 'main', + isLeader: true, }); beforeEach(async () => { @@ -67,7 +73,7 @@ describe('License', () => { license = new License( logger, - mock({ instanceType: 'worker' }), + mock({ instanceType: 'worker', isLeader: false }), mock(), mock(), mock({ license: licenseConfig }), @@ -134,7 +140,7 @@ describe('License', () => { expect(LicenseManager.prototype.getManagementJwt).toHaveBeenCalled(); }); - test('getMainPlan() returns the right entitlement', async () => { + test('getMainPlan() returns the latest main entitlement', async () => { // mock entitlements response License.prototype.getCurrentEntitlements = jest.fn().mockReturnValue([ { @@ -143,8 +149,21 @@ describe('License', () => { productMetadata: {}, features: {}, featureOverrides: {}, - validFrom: new Date(), - validTo: new Date(), + validFrom: makeDateWithHourOffset(-3), + validTo: makeDateWithHourOffset(1), + }, + { + id: '95b9c852-1349-478d-9ad1-b3f55510e488', + productId: '670650f2-72d8-4397-898c-c249906e2cc2', + productMetadata: { + terms: { + isMainPlan: true, + }, + }, + features: {}, + featureOverrides: {}, + validFrom: makeDateWithHourOffset(-2), + validTo: makeDateWithHourOffset(1), }, { id: MOCK_MAIN_PLAN_ID, @@ -156,8 +175,8 @@ describe('License', () => { }, features: {}, featureOverrides: {}, - validFrom: new Date(), - validTo: new Date(), + validFrom: makeDateWithHourOffset(-1), // this is the LATEST / newest plan + validTo: makeDateWithHourOffset(1), }, ]); jest.fn(license.getMainPlan).mockReset(); @@ -175,8 +194,8 @@ describe('License', () => { productMetadata: {}, // has no `productMetadata.terms.isMainPlan`! features: {}, featureOverrides: {}, - validFrom: new Date(), - validTo: new Date(), + validFrom: makeDateWithHourOffset(-1), + validTo: makeDateWithHourOffset(1), }, { id: 'c1aae471-c24e-4874-ad88-b97107de486c', @@ -184,8 +203,8 @@ describe('License', () => { productMetadata: {}, // has no `productMetadata.terms.isMainPlan`! features: {}, featureOverrides: {}, - validFrom: new Date(), - validTo: new Date(), + validFrom: makeDateWithHourOffset(-1), + validTo: makeDateWithHourOffset(1), }, ]); jest.fn(license.getMainPlan).mockReset(); @@ -197,94 +216,73 @@ describe('License', () => { describe('License', () => { describe('init', () => { - describe('in single-main setup', () => { - describe('with `license.autoRenewEnabled` enabled', () => { - it('should enable renewal', async () => { - const globalConfig = mock({ - license: licenseConfig, - multiMainSetup: { enabled: false }, - }); - - await new License( - mockLogger(), - mock({ instanceType: 'main' }), - mock(), - mock(), - globalConfig, - ).init(); - - expect(LicenseManager).toHaveBeenCalledWith( - expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), - ); - }); + it('when leader main with N8N_LICENSE_AUTO_RENEW_ENABLED=true, should enable renewal', async () => { + const globalConfig = mock({ + license: { ...licenseConfig, autoRenewalEnabled: true }, }); - describe('with `license.autoRenewEnabled` disabled', () => { - it('should disable renewal', async () => { - await new License( - mockLogger(), - mock({ instanceType: 'main' }), - mock(), - mock(), - mock(), - ).init(); + await new License( + mockLogger(), + mock({ instanceType: 'main', isLeader: true }), + mock(), + mock(), + globalConfig, + ).init(); - expect(LicenseManager).toHaveBeenCalledWith( - expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), - ); - }); - }); + expect(LicenseManager).toHaveBeenCalledWith( + expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), + ); }); - describe('in multi-main setup', () => { - describe('with `license.autoRenewEnabled` disabled', () => { - test.each(['unset', 'leader', 'follower'])( - 'if %s status, should disable removal', - async (status) => { - const globalConfig = mock({ - license: { ...licenseConfig, autoRenewalEnabled: false }, - multiMainSetup: { enabled: true }, - }); - config.set('multiMainSetup.instanceType', status); - - await new License(mockLogger(), mock(), mock(), mock(), globalConfig).init(); - - expect(LicenseManager).toHaveBeenCalledWith( - expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), - ); - }, - ); + it.each([ + { + scenario: 'when leader main with N8N_LICENSE_AUTO_RENEW_ENABLED=false', + isLeader: true, + autoRenewalEnabled: false, + }, + { + scenario: 'when follower main with N8N_LICENSE_AUTO_RENEW_ENABLED=true', + isLeader: false, + autoRenewalEnabled: true, + }, + { + scenario: 'when follower main with N8N_LICENSE_AUTO_RENEW_ENABLED=false', + isLeader: false, + autoRenewalEnabled: false, + }, + ])('$scenario, should disable renewal', async ({ isLeader, autoRenewalEnabled }) => { + const globalConfig = mock({ + license: { ...licenseConfig, autoRenewalEnabled }, }); - describe('with `license.autoRenewEnabled` enabled', () => { - test.each(['unset', 'follower'])('if %s status, should disable removal', async (status) => { - const globalConfig = mock({ - license: { ...licenseConfig, autoRenewalEnabled: false }, - multiMainSetup: { enabled: true }, - }); - config.set('multiMainSetup.instanceType', status); + await new License( + mockLogger(), + mock({ instanceType: 'main', isLeader }), + mock(), + mock(), + globalConfig, + ).init(); - await new License(mockLogger(), mock(), mock(), mock(), globalConfig).init(); + const expectedRenewalSettings = + isLeader && autoRenewalEnabled + ? { autoRenewEnabled: true, renewOnInit: true } + : { autoRenewEnabled: false, renewOnInit: false }; - expect(LicenseManager).toHaveBeenCalledWith( - expect.objectContaining({ autoRenewEnabled: false, renewOnInit: false }), - ); - }); + expect(LicenseManager).toHaveBeenCalledWith(expect.objectContaining(expectedRenewalSettings)); + }); - it('if leader status, should enable renewal', async () => { - const globalConfig = mock({ - license: licenseConfig, - multiMainSetup: { enabled: true }, - }); - config.set('multiMainSetup.instanceType', 'leader'); - - await new License(mockLogger(), mock(), mock(), mock(), globalConfig).init(); - - expect(LicenseManager).toHaveBeenCalledWith( - expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), - ); - }); + it('when CLI command with N8N_LICENSE_AUTO_RENEW_ENABLED=true, should enable renewal', async () => { + const globalConfig = mock({ + license: { ...licenseConfig, autoRenewalEnabled: true }, }); + + await new License(mockLogger(), mock(), mock(), mock(), globalConfig).init({ + isCli: true, + }); + + expect(LicenseManager).toHaveBeenCalledWith( + expect.objectContaining({ autoRenewEnabled: true, renewOnInit: true }), + ); }); }); @@ -297,7 +295,7 @@ describe('License', () => { await license.reinit(); - expect(initSpy).toHaveBeenCalledWith(true); + expect(initSpy).toHaveBeenCalledWith({ forceRecreate: true }); expect(LicenseManager.prototype.reset).toHaveBeenCalled(); expect(LicenseManager.prototype.initialize).toHaveBeenCalled(); diff --git a/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts b/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts index 75aa602301..ddc55ae25e 100644 --- a/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts +++ b/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts @@ -107,7 +107,7 @@ describe('LoadNodesAndCredentials', () => { }; fullNodeWrapper.description.properties = [existingProp]; const result = instance.convertNodeToAiTool(fullNodeWrapper); - expect(result.description.properties).toHaveLength(3); // Existing prop + toolDescription + notice + expect(result.description.properties).toHaveLength(2); // Existing prop + toolDescription expect(result.description.properties).toContainEqual(existingProp); }); @@ -121,9 +121,9 @@ describe('LoadNodesAndCredentials', () => { }; fullNodeWrapper.description.properties = [resourceProp]; const result = instance.convertNodeToAiTool(fullNodeWrapper); - expect(result.description.properties[1].name).toBe('descriptionType'); - expect(result.description.properties[2].name).toBe('toolDescription'); - expect(result.description.properties[3]).toEqual(resourceProp); + expect(result.description.properties[0].name).toBe('descriptionType'); + expect(result.description.properties[1].name).toBe('toolDescription'); + expect(result.description.properties[2]).toEqual(resourceProp); }); it('should handle nodes with operation property', () => { @@ -136,9 +136,9 @@ describe('LoadNodesAndCredentials', () => { }; fullNodeWrapper.description.properties = [operationProp]; const result = instance.convertNodeToAiTool(fullNodeWrapper); - expect(result.description.properties[1].name).toBe('descriptionType'); - expect(result.description.properties[2].name).toBe('toolDescription'); - expect(result.description.properties[3]).toEqual(operationProp); + expect(result.description.properties[0].name).toBe('descriptionType'); + expect(result.description.properties[1].name).toBe('toolDescription'); + expect(result.description.properties[2]).toEqual(operationProp); }); it('should handle nodes with both resource and operation properties', () => { @@ -158,17 +158,17 @@ describe('LoadNodesAndCredentials', () => { }; fullNodeWrapper.description.properties = [resourceProp, operationProp]; const result = instance.convertNodeToAiTool(fullNodeWrapper); - expect(result.description.properties[1].name).toBe('descriptionType'); - expect(result.description.properties[2].name).toBe('toolDescription'); - expect(result.description.properties[3]).toEqual(resourceProp); - expect(result.description.properties[4]).toEqual(operationProp); + expect(result.description.properties[0].name).toBe('descriptionType'); + expect(result.description.properties[1].name).toBe('toolDescription'); + expect(result.description.properties[2]).toEqual(resourceProp); + expect(result.description.properties[3]).toEqual(operationProp); }); it('should handle nodes with empty properties', () => { fullNodeWrapper.description.properties = []; const result = instance.convertNodeToAiTool(fullNodeWrapper); - expect(result.description.properties).toHaveLength(2); - expect(result.description.properties[1].name).toBe('toolDescription'); + expect(result.description.properties).toHaveLength(1); + expect(result.description.properties[0].name).toBe('toolDescription'); }); it('should handle nodes with existing codex property', () => { @@ -211,4 +211,44 @@ describe('LoadNodesAndCredentials', () => { expect(result.description.displayName).toBe('Special @#$% Node Tool'); }); }); + + describe('resolveSchema', () => { + let instance: LoadNodesAndCredentials; + + beforeEach(() => { + instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock()); + instance.knownNodes['n8n-nodes-base.test'] = { + className: 'Test', + sourcePath: '/nodes-base/dist/nodes/Test/Test.node.js', + }; + }); + + it('should return undefined if the node is not known', () => { + const result = instance.resolveSchema({ + node: 'n8n-nodes-base.doesNotExist', + version: '1.0.0', + resource: 'account', + operation: 'get', + }); + expect(result).toBeUndefined(); + }); + + it('should return the correct path if the node is known', () => { + const result = instance.resolveSchema({ + node: 'n8n-nodes-base.test', + version: '1.0.0', + resource: 'account', + operation: 'get', + }); + expect(result).toEqual('/nodes-base/dist/nodes/Test/__schema__/v1.0.0/account/get.json'); + }); + + it('should return the correct path if there is no resource or operation', () => { + const result = instance.resolveSchema({ + node: 'n8n-nodes-base.test', + version: '1.0.0', + }); + expect(result).toEqual('/nodes-base/dist/nodes/Test/__schema__/v1.0.0.json'); + }); + }); }); diff --git a/packages/cli/src/__tests__/manual-execution.service.test.ts b/packages/cli/src/__tests__/manual-execution.service.test.ts index 383a8dc87c..23f69ab821 100644 --- a/packages/cli/src/__tests__/manual-execution.service.test.ts +++ b/packages/cli/src/__tests__/manual-execution.service.test.ts @@ -46,5 +46,26 @@ describe('ManualExecutionService', () => { name: 'node2', }); }); + + it('Should return triggerToStartFrom trigger node', () => { + const data = { + pinData: { + node1: {}, + node2: {}, + }, + triggerToStartFrom: { name: 'node3' }, + } as unknown as IWorkflowExecutionDataProcess; + const workflow = { + getNode(nodeName: string) { + return { + name: nodeName, + }; + }, + } as unknown as Workflow; + const executionStartNode = manualExecutionService.getExecutionStartNode(data, workflow); + expect(executionStartNode).toEqual({ + name: 'node3', + }); + }); }); }); diff --git a/packages/cli/src/__tests__/node-types.test.ts b/packages/cli/src/__tests__/node-types.test.ts index 78d0c5e18a..5e26cc5665 100644 --- a/packages/cli/src/__tests__/node-types.test.ts +++ b/packages/cli/src/__tests__/node-types.test.ts @@ -1,5 +1,6 @@ +import type { GlobalConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; -import { UnrecognizedNodeTypeError } from 'n8n-core'; +import { RoutingNode, UnrecognizedNodeTypeError } from 'n8n-core'; import type { LoadedClass, INodeType, @@ -11,9 +12,14 @@ import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { NodeTypes } from '@/node-types'; describe('NodeTypes', () => { - const loadNodesAndCredentials = mock(); + const globalConfig = mock({ + nodes: { communityPackages: { allowToolUsage: false } }, + }); + const loadNodesAndCredentials = mock({ + convertNodeToAiTool: LoadNodesAndCredentials.prototype.convertNodeToAiTool, + }); - const nodeTypes: NodeTypes = new NodeTypes(loadNodesAndCredentials); + const nodeTypes: NodeTypes = new NodeTypes(globalConfig, loadNodesAndCredentials); const nonVersionedNode: LoadedClass = { sourcePath: '', @@ -22,10 +28,11 @@ describe('NodeTypes', () => { name: 'n8n-nodes-base.nonVersioned', usableAsTool: undefined, }), + supplyData: undefined, }, }; - const v1Node = mock(); - const v2Node = mock(); + const v1Node = mock({ supplyData: undefined }); + const v2Node = mock({ supplyData: undefined }); const versionedNode: LoadedClass = { sourcePath: '', type: { @@ -43,6 +50,17 @@ describe('NodeTypes', () => { }, }, }; + const toolNode: LoadedClass = { + sourcePath: '', + type: { + description: mock({ + name: 'n8n-nodes-base.toolNode', + displayName: 'TestNode', + properties: [], + }), + supplyData: jest.fn(), + }, + }; const toolSupportingNode: LoadedClass = { sourcePath: '', type: { @@ -52,19 +70,55 @@ describe('NodeTypes', () => { usableAsTool: true, properties: [], }), + supplyData: undefined, + }, + }; + const declarativeNode: LoadedClass = { + sourcePath: '', + type: { + description: mock({ + name: 'n8n-nodes-base.declarativeNode', + displayName: 'Declarative Node', + usableAsTool: true, + properties: [], + }), + execute: undefined, + poll: undefined, + trigger: undefined, + webhook: undefined, + methods: undefined, + supplyData: undefined, + }, + }; + const communityNode: LoadedClass = { + sourcePath: '', + type: { + description: mock({ + name: 'n8n-nodes-community.testNode', + displayName: 'TestNode', + usableAsTool: true, + properties: [], + }), + supplyData: undefined, }, }; loadNodesAndCredentials.getNode.mockImplementation((fullNodeType) => { const [packageName, nodeType] = fullNodeType.split('.'); - if (nodeType === 'nonVersioned') return nonVersionedNode; - if (nodeType === 'versioned') return versionedNode; - if (nodeType === 'testNode') return toolSupportingNode; + if (packageName === 'n8n-nodes-base') { + if (nodeType === 'nonVersioned') return nonVersionedNode; + if (nodeType === 'versioned') return versionedNode; + if (nodeType === 'testNode') return toolSupportingNode; + if (nodeType === 'declarativeNode') return declarativeNode; + if (nodeType === 'toolNode') return toolNode; + } else if (fullNodeType === 'n8n-nodes-community.testNode') return communityNode; throw new UnrecognizedNodeTypeError(packageName, nodeType); }); beforeEach(() => { jest.clearAllMocks(); + globalConfig.nodes.communityPackages.allowToolUsage = false; + loadNodesAndCredentials.loaded.nodes = {}; }); describe('getByName', () => { @@ -103,18 +157,63 @@ describe('NodeTypes', () => { ); }); + it('should throw when a node-type is requested as tool, but the original node is already a tool', () => { + expect(() => nodeTypes.getByNameAndVersion('n8n-nodes-base.toolNodeTool')).toThrow( + 'Node already has a `supplyData` method', + ); + }); + it('should return the tool node-type when requested as tool', () => { - // @ts-expect-error don't mock convertNodeToAiTool for now - loadNodesAndCredentials.convertNodeToAiTool = - LoadNodesAndCredentials.prototype.convertNodeToAiTool; const result = nodeTypes.getByNameAndVersion('n8n-nodes-base.testNodeTool'); - expect(result).not.toEqual(toolSupportingNode); + expect(result).not.toEqual(toolSupportingNode.type); expect(result.description.name).toEqual('n8n-nodes-base.testNodeTool'); expect(result.description.displayName).toEqual('TestNode Tool'); expect(result.description.codex?.categories).toContain('AI'); expect(result.description.inputs).toEqual([]); expect(result.description.outputs).toEqual(['ai_tool']); }); + + it('should throw when a node-type is requested as tool, but is a community package', () => { + expect(() => nodeTypes.getByNameAndVersion('n8n-nodes-community.testNodeTool')).toThrow( + 'Unrecognized node type: n8n-nodes-community.testNodeTool', + ); + }); + + it('should return a tool node-type from a community node, when requested as tool', () => { + globalConfig.nodes.communityPackages.allowToolUsage = true; + const result = nodeTypes.getByNameAndVersion('n8n-nodes-community.testNodeTool'); + expect(result).not.toEqual(toolSupportingNode.type); + expect(result.description.name).toEqual('n8n-nodes-community.testNodeTool'); + expect(result.description.displayName).toEqual('TestNode Tool'); + expect(result.description.codex?.categories).toContain('AI'); + expect(result.description.inputs).toEqual([]); + expect(result.description.outputs).toEqual(['ai_tool']); + }); + + it('should return a declarative node-type with an `.execute` method', () => { + const result = nodeTypes.getByNameAndVersion('n8n-nodes-base.declarativeNode'); + expect(result).toBe(declarativeNode.type); + expect(result.execute).toBeDefined(); + + const runNodeSpy = jest.spyOn(RoutingNode.prototype, 'runNode').mockResolvedValue([]); + result.execute!.call(mock()); + expect(runNodeSpy).toHaveBeenCalled(); + }); + + it('should return a declarative node-type as a tool with an `.execute` method', () => { + const result = nodeTypes.getByNameAndVersion('n8n-nodes-base.declarativeNodeTool'); + expect(result).not.toEqual(declarativeNode.type); + expect(result.description.name).toEqual('n8n-nodes-base.declarativeNodeTool'); + expect(result.description.displayName).toEqual('Declarative Node Tool'); + expect(result.description.codex?.categories).toContain('AI'); + expect(result.description.inputs).toEqual([]); + expect(result.description.outputs).toEqual(['ai_tool']); + expect(result.execute).toBeDefined(); + + const runNodeSpy = jest.spyOn(RoutingNode.prototype, 'runNode').mockResolvedValue([]); + result.execute!.call(mock()); + expect(runNodeSpy).toHaveBeenCalled(); + }); }); describe('getWithSourcePath', () => { @@ -143,7 +242,6 @@ describe('NodeTypes', () => { describe('getNodeTypeDescriptions', () => { it('should return descriptions for valid node types', () => { - const nodeTypes = new NodeTypes(loadNodesAndCredentials); const result = nodeTypes.getNodeTypeDescriptions([ { name: 'n8n-nodes-base.nonVersioned', version: 1 }, ]); @@ -153,7 +251,6 @@ describe('NodeTypes', () => { }); it('should throw error for invalid node type', () => { - const nodeTypes = new NodeTypes(loadNodesAndCredentials); expect(() => nodeTypes.getNodeTypeDescriptions([{ name: 'n8n-nodes-base.nonExistent', version: 1 }]), ).toThrow('Unrecognized node type: n8n-nodes-base.nonExistent'); diff --git a/packages/cli/src/__tests__/project.test-data.ts b/packages/cli/src/__tests__/project.test-data.ts index 3ffac36fc8..cd176bd291 100644 --- a/packages/cli/src/__tests__/project.test-data.ts +++ b/packages/cli/src/__tests__/project.test-data.ts @@ -1,7 +1,7 @@ import { nanoId, date, firstName, lastName, email } from 'minifaker'; import 'minifaker/locales/en'; -import type { Project, ProjectType } from '@/databases/entities/project'; +import type { Project } from '@/databases/entities/project'; type RawProjectData = Pick; @@ -13,7 +13,7 @@ export const createRawProjectData = (payload: Partial): Project updatedAt: date(), id: nanoId.nanoid(), name: projectName, - type: 'personal' as ProjectType, + type: 'personal', ...payload, } as Project; }; diff --git a/packages/cli/src/__tests__/workflow-runner.test.ts b/packages/cli/src/__tests__/workflow-runner.test.ts index 7330f85997..1de4ddef6b 100644 --- a/packages/cli/src/__tests__/workflow-runner.test.ts +++ b/packages/cli/src/__tests__/workflow-runner.test.ts @@ -12,17 +12,14 @@ import type { IWorkflowExecutionDataProcess, StartNodeData, } from 'n8n-workflow'; -import { - Workflow, - WorkflowHooks, - type ExecutionError, - type IWorkflowExecuteHooks, -} from 'n8n-workflow'; +import { Workflow, type ExecutionError } from 'n8n-workflow'; import PCancelable from 'p-cancelable'; import { ActiveExecutions } from '@/active-executions'; import config from '@/config'; +import type { ExecutionEntity } from '@/databases/entities/execution-entity'; import type { User } from '@/databases/entities/user'; +import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ExecutionNotFoundError } from '@/errors/execution-not-found-error'; import { Telemetry } from '@/telemetry'; import { PermissionChecker } from '@/user-management/permission-checker'; @@ -36,25 +33,14 @@ import { setupTestServer } from '@test-integration/utils'; let owner: User; let runner: WorkflowRunner; -let hookFunctions: IWorkflowExecuteHooks; setupTestServer({ endpointGroups: [] }); mockInstance(Telemetry); -class Watchers { - workflowExecuteAfter = jest.fn(); -} -const watchers = new Watchers(); -const watchedWorkflowExecuteAfter = jest.spyOn(watchers, 'workflowExecuteAfter'); - beforeAll(async () => { owner = await createUser({ role: 'global:owner' }); runner = Container.get(WorkflowRunner); - - hookFunctions = { - workflowExecuteAfter: [watchers.workflowExecuteAfter], - }; }); afterAll(() => { @@ -67,6 +53,20 @@ beforeEach(async () => { }); describe('processError', () => { + let workflow: WorkflowEntity; + let execution: ExecutionEntity; + let hooks: core.ExecutionLifecycleHooks; + + const watcher = mock<{ workflowExecuteAfter: () => Promise }>(); + + beforeEach(async () => { + jest.clearAllMocks(); + workflow = await createWorkflow({}, owner); + execution = await createExecution({ status: 'success', finished: true }, workflow); + hooks = new core.ExecutionLifecycleHooks('webhook', execution.id, workflow); + hooks.addHandler('workflowExecuteAfter', watcher.workflowExecuteAfter); + }); + test('processError should return early in Bull stalled edge case', async () => { const workflow = await createWorkflow({}, owner); const execution = await createExecution( @@ -82,9 +82,9 @@ describe('processError', () => { new Date(), 'webhook', execution.id, - new WorkflowHooks(hookFunctions, 'webhook', execution.id, workflow), + hooks, ); - expect(watchedWorkflowExecuteAfter).toHaveBeenCalledTimes(0); + expect(watcher.workflowExecuteAfter).toHaveBeenCalledTimes(0); }); test('processError should return early if the error is `ExecutionNotFoundError`', async () => { @@ -95,9 +95,9 @@ describe('processError', () => { new Date(), 'webhook', execution.id, - new WorkflowHooks(hookFunctions, 'webhook', execution.id, workflow), + hooks, ); - expect(watchedWorkflowExecuteAfter).toHaveBeenCalledTimes(0); + expect(watcher.workflowExecuteAfter).toHaveBeenCalledTimes(0); }); test('processError should process error', async () => { @@ -119,9 +119,9 @@ describe('processError', () => { new Date(), 'webhook', execution.id, - new WorkflowHooks(hookFunctions, 'webhook', execution.id, workflow), + hooks, ); - expect(watchedWorkflowExecuteAfter).toHaveBeenCalledTimes(1); + expect(watcher.workflowExecuteAfter).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/cli/src/active-executions.ts b/packages/cli/src/active-executions.ts index 9056271f64..e0dde622eb 100644 --- a/packages/cli/src/active-executions.ts +++ b/packages/cli/src/active-executions.ts @@ -61,9 +61,7 @@ export class ActiveExecutions { workflowId: executionData.workflowData.id, }; - if (executionData.retryOf !== undefined) { - fullExecutionData.retryOf = executionData.retryOf.toString(); - } + fullExecutionData.retryOf = executionData.retryOf ?? undefined; const workflowId = executionData.workflowData.id; if (workflowId !== undefined && isWorkflowIdValid(workflowId)) { @@ -94,14 +92,17 @@ export class ActiveExecutions { await this.executionRepository.updateExistingExecution(executionId, execution); } + const resumingExecution = this.activeExecutions[executionId]; const postExecutePromise = createDeferredPromise(); - this.activeExecutions[executionId] = { + const execution: IExecutingWorkflowData = { executionData, - startedAt: new Date(), + startedAt: resumingExecution?.startedAt ?? new Date(), postExecutePromise, status: executionStatus, + responsePromise: resumingExecution?.responsePromise, }; + this.activeExecutions[executionId] = execution; // Automatically remove execution once the postExecutePromise settles void postExecutePromise.promise @@ -111,8 +112,13 @@ export class ActiveExecutions { }) .finally(() => { this.concurrencyControl.release({ mode: executionData.executionMode }); - delete this.activeExecutions[executionId]; - this.logger.debug('Execution removed', { executionId }); + if (execution.status === 'waiting') { + // Do not hold on a reference to the previous WorkflowExecute instance, since a resuming execution will use a new instance + delete execution.workflowExecution; + } else { + delete this.activeExecutions[executionId]; + this.logger.debug('Execution removed', { executionId }); + } }); this.logger.debug('Execution added', { executionId }); @@ -125,14 +131,14 @@ export class ActiveExecutions { */ attachWorkflowExecution(executionId: string, workflowExecution: PCancelable) { - this.getExecution(executionId).workflowExecution = workflowExecution; + this.getExecutionOrFail(executionId).workflowExecution = workflowExecution; } attachResponsePromise( executionId: string, responsePromise: IDeferredPromise, ): void { - this.getExecution(executionId).responsePromise = responsePromise; + this.getExecutionOrFail(executionId).responsePromise = responsePromise; } resolveResponsePromise(executionId: string, response: IExecuteResponsePromiseData): void { @@ -147,15 +153,23 @@ export class ActiveExecutions { // There is no execution running with that id return; } - execution.workflowExecution?.cancel(); - execution.postExecutePromise.reject(new ExecutionCancelledError(executionId)); + const error = new ExecutionCancelledError(executionId); + execution.responsePromise?.reject(error); + if (execution.status === 'waiting') { + // A waiting execution will not have a valid workflowExecution or postExecutePromise + // So we can't rely on the `.finally` on the postExecutePromise for the execution removal + delete this.activeExecutions[executionId]; + } else { + execution.workflowExecution?.cancel(); + execution.postExecutePromise.reject(error); + } this.logger.debug('Execution cancelled', { executionId }); } /** Resolve the post-execution promise in an execution. */ finalizeExecution(executionId: string, fullRunData?: IRun) { if (!this.has(executionId)) return; - const execution = this.getExecution(executionId); + const execution = this.getExecutionOrFail(executionId); execution.postExecutePromise.resolve(fullRunData); this.logger.debug('Execution finalized', { executionId }); } @@ -164,7 +178,7 @@ export class ActiveExecutions { * Returns a promise which will resolve with the data of the execution with the given id */ async getPostExecutePromise(executionId: string): Promise { - return await this.getExecution(executionId).postExecutePromise.promise; + return await this.getExecutionOrFail(executionId).postExecutePromise.promise; } /** @@ -179,7 +193,7 @@ export class ActiveExecutions { data = this.activeExecutions[id]; returnData.push({ id, - retryOf: data.executionData.retryOf, + retryOf: data.executionData.retryOf ?? undefined, startedAt: data.startedAt, mode: data.executionData.executionMode, workflowId: data.executionData.workflowData.id, @@ -191,32 +205,40 @@ export class ActiveExecutions { } setStatus(executionId: string, status: ExecutionStatus) { - this.getExecution(executionId).status = status; + this.getExecutionOrFail(executionId).status = status; } getStatus(executionId: string): ExecutionStatus { - return this.getExecution(executionId).status; + return this.getExecutionOrFail(executionId).status; } /** Wait for all active executions to finish */ async shutdown(cancelAll = false) { - let executionIds = Object.keys(this.activeExecutions); - - if (config.getEnv('executions.mode') === 'regular') { + const isRegularMode = config.getEnv('executions.mode') === 'regular'; + if (isRegularMode) { // removal of active executions will no longer release capacity back, // so that throttled executions cannot resume during shutdown this.concurrencyControl.disable(); } - if (cancelAll) { - if (config.getEnv('executions.mode') === 'regular') { - await this.concurrencyControl.removeAll(this.activeExecutions); + let executionIds = Object.keys(this.activeExecutions); + const toCancel: string[] = []; + for (const executionId of executionIds) { + const { responsePromise, status } = this.activeExecutions[executionId]; + if (!!responsePromise || (isRegularMode && cancelAll)) { + // Cancel all exectutions that have a response promise, because these promises can't be retained between restarts + this.stopExecution(executionId); + toCancel.push(executionId); + } else if (status === 'waiting' || status === 'new') { + // Remove waiting and new executions to not block shutdown + delete this.activeExecutions[executionId]; } - - executionIds.forEach((executionId) => this.stopExecution(executionId)); } + await this.concurrencyControl.removeAll(toCancel); + let count = 0; + executionIds = Object.keys(this.activeExecutions); while (executionIds.length !== 0) { if (count++ % 4 === 0) { this.logger.info(`Waiting for ${executionIds.length} active executions to finish...`); @@ -227,7 +249,7 @@ export class ActiveExecutions { } } - private getExecution(executionId: string): IExecutingWorkflowData { + getExecutionOrFail(executionId: string): IExecutingWorkflowData { const execution = this.activeExecutions[executionId]; if (!execution) { throw new ExecutionNotFoundError(executionId); diff --git a/packages/cli/src/active-workflow-manager.ts b/packages/cli/src/active-workflow-manager.ts index a002bc4054..b783724bef 100644 --- a/packages/cli/src/active-workflow-manager.ts +++ b/packages/cli/src/active-workflow-manager.ts @@ -7,13 +7,13 @@ import { Logger, PollContext, TriggerContext, + type IGetExecutePollFunctions, + type IGetExecuteTriggerFunctions, } from 'n8n-core'; import type { ExecutionError, IDeferredPromise, IExecuteResponsePromiseData, - IGetExecutePollFunctions, - IGetExecuteTriggerFunctions, INode, INodeExecutionData, IRun, @@ -40,6 +40,7 @@ import { import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { OnShutdown } from '@/decorators/on-shutdown'; +import { executeErrorWorkflow } from '@/execution-lifecycle/execute-error-workflow'; import { ExecutionService } from '@/executions/execution.service'; import { ExternalHooks } from '@/external-hooks'; import type { IWorkflowDb } from '@/interfaces'; @@ -88,7 +89,7 @@ export class ActiveWorkflowManager { await this.addActiveWorkflows('init'); - await this.externalHooks.run('activeWorkflows.initialized', []); + await this.externalHooks.run('activeWorkflows.initialized'); await this.webhookService.populateCache(); } @@ -400,7 +401,7 @@ export class ActiveWorkflowManager { status: 'running', }; - WorkflowExecuteAdditionalData.executeErrorWorkflow(workflowData, fullRunData, mode); + executeErrorWorkflow(workflowData, fullRunData, mode); } /** diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 56799c5089..5e07b8e350 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -61,10 +61,15 @@ export abstract class BaseCommand extends Command { async init(): Promise { this.errorReporter = Container.get(ErrorReporter); - await this.errorReporter.init( - this.instanceSettings.instanceType, - this.globalConfig.sentry.backendDsn, - ); + + const { backendDsn, n8nVersion, environment, deploymentName } = this.globalConfig.sentry; + await this.errorReporter.init({ + serverType: this.instanceSettings.instanceType, + dsn: backendDsn, + environment, + release: n8nVersion, + serverName: deploymentName, + }); initExpressionEvaluator(); process.once('SIGTERM', this.onTerminationSignal('SIGTERM')); @@ -184,42 +189,10 @@ export abstract class BaseCommand extends Command { private async _initObjectStoreService(options = { isReadOnly: false }) { const objectStoreService = Container.get(ObjectStoreService); - const { host, bucket, credentials } = this.globalConfig.externalStorage.s3; - - if (host === '') { - throw new ApplicationError( - 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', - ); - } - - if (bucket.name === '') { - throw new ApplicationError( - 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', - ); - } - - if (bucket.region === '') { - throw new ApplicationError( - 'External storage bucket region not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION`.', - ); - } - - if (credentials.accessKey === '') { - throw new ApplicationError( - 'External storage access key not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY`.', - ); - } - - if (credentials.accessSecret === '') { - throw new ApplicationError( - 'External storage access secret not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET`.', - ); - } - this.logger.debug('Initializing object store service'); try { - await objectStoreService.init(host, bucket, credentials); + await objectStoreService.init(); objectStoreService.setReadonly(options.isReadOnly); this.logger.debug('Object store init completed'); diff --git a/packages/cli/src/commands/ldap/reset.ts b/packages/cli/src/commands/ldap/reset.ts index edbf988434..8d3eb8da94 100644 --- a/packages/cli/src/commands/ldap/reset.ts +++ b/packages/cli/src/commands/ldap/reset.ts @@ -110,7 +110,7 @@ export class Reset extends BaseCommand { } for (const credential of ownedCredentials) { - await Container.get(CredentialsService).delete(credential); + await Container.get(CredentialsService).delete(owner, credential.id); } await Container.get(AuthProviderSyncHistoryRepository).delete({ providerType: 'ldap' }); diff --git a/packages/cli/src/commands/license/clear.ts b/packages/cli/src/commands/license/clear.ts index 03a2ea4dd4..732401ed47 100644 --- a/packages/cli/src/commands/license/clear.ts +++ b/packages/cli/src/commands/license/clear.ts @@ -16,7 +16,7 @@ export class ClearLicenseCommand extends BaseCommand { // Attempt to invoke shutdown() to force any floating entitlements to be released const license = Container.get(License); - await license.init(); + await license.init({ isCli: true }); try { await license.shutdown(); } catch { diff --git a/packages/cli/src/commands/license/info.ts b/packages/cli/src/commands/license/info.ts index cc99e925f7..f99648d0d5 100644 --- a/packages/cli/src/commands/license/info.ts +++ b/packages/cli/src/commands/license/info.ts @@ -11,7 +11,7 @@ export class LicenseInfoCommand extends BaseCommand { async run() { const license = Container.get(License); - await license.init(); + await license.init({ isCli: true }); this.logger.info('Printing license information:\n' + license.getInfo()); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 179c3a8052..a255a95bb9 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; import { Flags } from '@oclif/core'; import glob from 'fast-glob'; @@ -21,7 +20,6 @@ import { FeatureNotLicensedError } from '@/errors/feature-not-licensed.error'; import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { EventService } from '@/events/event.service'; import { ExecutionService } from '@/executions/execution.service'; -import { License } from '@/license'; import { PubSubHandler } from '@/scaling/pubsub/pubsub-handler'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import { Server } from '@/server'; @@ -96,7 +94,7 @@ export class Start extends BaseCommand { Container.get(WaitTracker).stopTracking(); - await this.externalHooks?.run('n8n.stop', []); + await this.externalHooks?.run('n8n.stop'); await this.activeWorkflowManager.removeAllTriggerAndPollerBasedWorkflows(); @@ -192,18 +190,23 @@ export class Start extends BaseCommand { await super.init(); this.activeWorkflowManager = Container.get(ActiveWorkflowManager); - this.instanceSettings.setMultiMainEnabled( - config.getEnv('executions.mode') === 'queue' && this.globalConfig.multiMainSetup.enabled, - ); - await this.initLicense(); + const isMultiMainEnabled = + config.getEnv('executions.mode') === 'queue' && this.globalConfig.multiMainSetup.enabled; + + this.instanceSettings.setMultiMainEnabled(isMultiMainEnabled); + + /** + * We temporarily license multi-main to allow orchestration to set instance + * role, which is needed by license init. Once the license is initialized, + * the actual value will be used for the license check. + */ + if (isMultiMainEnabled) this.instanceSettings.setMultiMainLicensed(true); await this.initOrchestration(); - this.logger.debug('Orchestration init complete'); + await this.initLicense(); - if (!this.globalConfig.license.autoRenewalEnabled && this.instanceSettings.isLeader) { - this.logger.warn( - 'Automatic license renewal is disabled. The license will not renew automatically, and access to licensed features may be lost!', - ); + if (isMultiMainEnabled && !this.license.isMultiMainLicensed()) { + throw new FeatureNotLicensedError(LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES); } Container.get(WaitTracker).init(); @@ -237,13 +240,6 @@ export class Start extends BaseCommand { return; } - if ( - Container.get(GlobalConfig).multiMainSetup.enabled && - !Container.get(License).isMultipleMainInstancesLicensed() - ) { - throw new FeatureNotLicensedError(LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES); - } - const orchestrationService = Container.get(OrchestrationService); await orchestrationService.init(); diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 8b6f318576..fd1e961b59 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -33,7 +33,7 @@ export class Webhook extends BaseCommand { this.logger.info('\nStopping n8n...'); try { - await this.externalHooks?.run('n8n.stop', []); + await this.externalHooks?.run('n8n.stop'); await Container.get(ActiveExecutions).shutdown(); } catch (error) { diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index f5138f1ef3..c6046a7772 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -49,7 +49,7 @@ export class Worker extends BaseCommand { this.logger.info('Stopping worker...'); try { - await this.externalHooks?.run('n8n.stop', []); + await this.externalHooks?.run('n8n.stop'); } catch (error) { await this.exitWithCrash('Error shutting down worker', error); } diff --git a/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts b/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts index 6511ae4d03..6a0ffcac18 100644 --- a/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts +++ b/packages/cli/src/concurrency/__tests__/concurrency-control.service.test.ts @@ -1,6 +1,7 @@ import { mock } from 'jest-mock-extended'; import type { WorkflowExecuteMode as ExecutionMode } from 'n8n-workflow'; +import type { ConcurrencyQueueType } from '@/concurrency/concurrency-control.service'; import { CLOUD_TEMP_PRODUCTION_LIMIT, CLOUD_TEMP_REPORTABLE_THRESHOLDS, @@ -10,7 +11,6 @@ import config from '@/config'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { InvalidConcurrencyLimitError } from '@/errors/invalid-concurrency-limit.error'; import type { EventService } from '@/events/event.service'; -import type { IExecutingWorkflowData } from '@/interfaces'; import type { Telemetry } from '@/telemetry'; import { mockLogger } from '@test/mocking'; @@ -24,61 +24,71 @@ describe('ConcurrencyControlService', () => { afterEach(() => { config.set('executions.concurrency.productionLimit', -1); + config.set('executions.concurrency.evaluationLimit', -1); config.set('executions.mode', 'integrated'); jest.clearAllMocks(); }); describe('constructor', () => { - it('should be enabled if production cap is positive', () => { - /** - * Arrange - */ - config.set('executions.concurrency.productionLimit', 1); + it.each(['production', 'evaluation'])( + 'should be enabled if %s cap is positive', + (type: ConcurrencyQueueType) => { + /** + * Arrange + */ + config.set(`executions.concurrency.${type}Limit`, 1); - /** - * Act - */ - const service = new ConcurrencyControlService( - logger, - executionRepository, - telemetry, - eventService, - ); - - /** - * Assert - */ - // @ts-expect-error Private property - expect(service.isEnabled).toBe(true); - // @ts-expect-error Private property - expect(service.productionQueue).toBeDefined(); - }); - - it('should throw if production cap is 0', () => { - /** - * Arrange - */ - config.set('executions.concurrency.productionLimit', 0); - - try { /** * Act */ - new ConcurrencyControlService(logger, executionRepository, telemetry, eventService); - } catch (error) { + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + /** * Assert */ - expect(error).toBeInstanceOf(InvalidConcurrencyLimitError); - } - }); + // @ts-expect-error Private property + expect(service.isEnabled).toBe(true); + // @ts-expect-error Private property + expect(service.queues.get(type)).toBeDefined(); + // @ts-expect-error Private property + expect(service.queues.size).toBe(1); + }, + ); - it('should be disabled if production cap is -1', () => { + it.each(['production', 'evaluation'])( + 'should throw if %s cap is 0', + (type: ConcurrencyQueueType) => { + /** + * Arrange + */ + config.set(`executions.concurrency.${type}Limit`, 0); + + try { + /** + * Act + */ + new ConcurrencyControlService(logger, executionRepository, telemetry, eventService); + } catch (error) { + /** + * Assert + */ + expect(error).toBeInstanceOf(InvalidConcurrencyLimitError); + } + }, + ); + + it('should be disabled if both production and evaluation caps are -1', () => { /** * Arrange */ config.set('executions.concurrency.productionLimit', -1); + config.set('executions.concurrency.evaluationLimit', -1); /** * Act @@ -97,28 +107,31 @@ describe('ConcurrencyControlService', () => { expect(service.isEnabled).toBe(false); }); - it('should be disabled if production cap is lower than -1', () => { - /** - * Arrange - */ - config.set('executions.concurrency.productionLimit', -2); + it.each(['production', 'evaluation'])( + 'should be disabled if %s cap is lower than -1', + (type: ConcurrencyQueueType) => { + /** + * Arrange + */ + config.set(`executions.concurrency.${type}Limit`, -2); - /** - * Act - */ - const service = new ConcurrencyControlService( - logger, - executionRepository, - telemetry, - eventService, - ); + /** + * Act + */ + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); - /** - * Act - */ - // @ts-expect-error Private property - expect(service.isEnabled).toBe(false); - }); + /** + * Act + */ + // @ts-expect-error Private property + expect(service.isEnabled).toBe(false); + }, + ); it('should be disabled on queue mode', () => { /** @@ -203,6 +216,31 @@ describe('ConcurrencyControlService', () => { */ expect(enqueueSpy).toHaveBeenCalled(); }); + + it('should enqueue on evaluation mode', async () => { + /** + * Arrange + */ + config.set('executions.concurrency.evaluationLimit', 1); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + const enqueueSpy = jest.spyOn(ConcurrencyQueue.prototype, 'enqueue'); + + /** + * Act + */ + await service.throttle({ mode: 'evaluation', executionId: '1' }); + + /** + * Assert + */ + expect(enqueueSpy).toHaveBeenCalled(); + }); }); describe('release', () => { @@ -258,6 +296,31 @@ describe('ConcurrencyControlService', () => { */ expect(dequeueSpy).toHaveBeenCalled(); }); + + it('should dequeue on evaluation mode', () => { + /** + * Arrange + */ + config.set('executions.concurrency.evaluationLimit', 1); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + const dequeueSpy = jest.spyOn(ConcurrencyQueue.prototype, 'dequeue'); + + /** + * Act + */ + service.release({ mode: 'evaluation' }); + + /** + * Assert + */ + expect(dequeueSpy).toHaveBeenCalled(); + }); }); describe('remove', () => { @@ -316,14 +379,12 @@ describe('ConcurrencyControlService', () => { expect(removeSpy).toHaveBeenCalled(); }, ); - }); - describe('removeAll', () => { - it('should remove all executions from the production queue', async () => { + it('should remove an execution on evaluation mode', () => { /** * Arrange */ - config.set('executions.concurrency.productionLimit', 2); + config.set('executions.concurrency.evaluationLimit', 1); const service = new ConcurrencyControlService( logger, @@ -331,28 +392,108 @@ describe('ConcurrencyControlService', () => { telemetry, eventService, ); - - jest - .spyOn(ConcurrencyQueue.prototype, 'getAll') - .mockReturnValueOnce(new Set(['1', '2', '3'])); - const removeSpy = jest.spyOn(ConcurrencyQueue.prototype, 'remove'); /** * Act */ - await service.removeAll({ - '1': mock(), - '2': mock(), - '3': mock(), - }); + service.remove({ mode: 'evaluation', executionId: '1' }); /** * Assert */ - expect(removeSpy).toHaveBeenNthCalledWith(1, '1'); - expect(removeSpy).toHaveBeenNthCalledWith(2, '2'); - expect(removeSpy).toHaveBeenNthCalledWith(3, '3'); + expect(removeSpy).toHaveBeenCalled(); + }); + }); + + describe('removeAll', () => { + it.each(['production', 'evaluation'])( + 'should remove all executions from the %s queue', + async (type: ConcurrencyQueueType) => { + /** + * Arrange + */ + config.set(`executions.concurrency.${type}Limit`, 2); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + + jest + .spyOn(ConcurrencyQueue.prototype, 'getAll') + .mockReturnValueOnce(new Set(['1', '2', '3'])); + + const removeSpy = jest.spyOn(ConcurrencyQueue.prototype, 'remove'); + + /** + * Act + */ + await service.removeAll(['1', '2', '3']); + + /** + * Assert + */ + expect(removeSpy).toHaveBeenNthCalledWith(1, '1'); + expect(removeSpy).toHaveBeenNthCalledWith(2, '2'); + expect(removeSpy).toHaveBeenNthCalledWith(3, '3'); + }, + ); + }); + + describe('get queue', () => { + it('should choose the production queue', async () => { + /** + * Arrange + */ + config.set('executions.concurrency.productionLimit', 2); + config.set('executions.concurrency.evaluationLimit', 2); + + /** + * Act + */ + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + // @ts-expect-error Private property + const queue = service.getQueue('webhook'); + + /** + * Assert + */ + // @ts-expect-error Private property + expect(queue).toEqual(service.queues.get('production')); + }); + + it('should choose the evaluation queue', async () => { + /** + * Arrange + */ + config.set('executions.concurrency.productionLimit', 2); + config.set('executions.concurrency.evaluationLimit', 2); + + /** + * Act + */ + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + // @ts-expect-error Private property + const queue = service.getQueue('evaluation'); + + /** + * Assert + */ + // @ts-expect-error Private property + expect(queue).toEqual(service.queues.get('evaluation')); }); }); }); @@ -388,6 +529,32 @@ describe('ConcurrencyControlService', () => { */ expect(enqueueSpy).not.toHaveBeenCalled(); }); + + it('should do nothing for evaluation executions', async () => { + /** + * Arrange + */ + config.set('executions.concurrency.evaluationLimit', -1); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + const enqueueSpy = jest.spyOn(ConcurrencyQueue.prototype, 'enqueue'); + + /** + * Act + */ + await service.throttle({ mode: 'evaluation', executionId: '1' }); + await service.throttle({ mode: 'evaluation', executionId: '2' }); + + /** + * Assert + */ + expect(enqueueSpy).not.toHaveBeenCalled(); + }); }); describe('release', () => { @@ -415,6 +582,31 @@ describe('ConcurrencyControlService', () => { */ expect(dequeueSpy).not.toHaveBeenCalled(); }); + + it('should do nothing for evaluation executions', () => { + /** + * Arrange + */ + config.set('executions.concurrency.evaluationLimit', -1); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + const dequeueSpy = jest.spyOn(ConcurrencyQueue.prototype, 'dequeue'); + + /** + * Act + */ + service.release({ mode: 'evaluation' }); + + /** + * Assert + */ + expect(dequeueSpy).not.toHaveBeenCalled(); + }); }); describe('remove', () => { @@ -442,6 +634,31 @@ describe('ConcurrencyControlService', () => { */ expect(removeSpy).not.toHaveBeenCalled(); }); + + it('should do nothing for evaluation executions', () => { + /** + * Arrange + */ + config.set('executions.concurrency.evaluationLimit', -1); + + const service = new ConcurrencyControlService( + logger, + executionRepository, + telemetry, + eventService, + ); + const removeSpy = jest.spyOn(ConcurrencyQueue.prototype, 'remove'); + + /** + * Act + */ + service.remove({ mode: 'evaluation', executionId: '1' }); + + /** + * Assert + */ + expect(removeSpy).not.toHaveBeenCalled(); + }); }); }); @@ -470,14 +687,17 @@ describe('ConcurrencyControlService', () => { * Act */ // @ts-expect-error Private property - service.productionQueue.emit('concurrency-check', { + service.queues.get('production').emit('concurrency-check', { capacity: CLOUD_TEMP_PRODUCTION_LIMIT - threshold, }); /** * Assert */ - expect(telemetry.track).toHaveBeenCalledWith('User hit concurrency limit', { threshold }); + expect(telemetry.track).toHaveBeenCalledWith('User hit concurrency limit', { + threshold, + concurrencyQueue: 'production', + }); }, ); @@ -500,7 +720,7 @@ describe('ConcurrencyControlService', () => { * Act */ // @ts-expect-error Private property - service.productionQueue.emit('concurrency-check', { + service.queues.get('production').emit('concurrency-check', { capacity: CLOUD_TEMP_PRODUCTION_LIMIT - threshold, }); @@ -532,7 +752,7 @@ describe('ConcurrencyControlService', () => { * Act */ // @ts-expect-error Private property - service.productionQueue.emit('concurrency-check', { + service.queues.get('production').emit('concurrency-check', { capacity: CLOUD_TEMP_PRODUCTION_LIMIT - threshold, }); diff --git a/packages/cli/src/concurrency/concurrency-control.service.ts b/packages/cli/src/concurrency/concurrency-control.service.ts index 6088d0f4c3..f41a919680 100644 --- a/packages/cli/src/concurrency/concurrency-control.service.ts +++ b/packages/cli/src/concurrency/concurrency-control.service.ts @@ -1,4 +1,5 @@ import { Service } from '@n8n/di'; +import { capitalize } from 'lodash'; import { Logger } from 'n8n-core'; import type { WorkflowExecuteMode as ExecutionMode } from 'n8n-workflow'; @@ -7,7 +8,6 @@ import { ExecutionRepository } from '@/databases/repositories/execution.reposito import { InvalidConcurrencyLimitError } from '@/errors/invalid-concurrency-limit.error'; import { UnknownExecutionModeError } from '@/errors/unknown-execution-mode.error'; import { EventService } from '@/events/event.service'; -import type { IExecutingWorkflowData } from '@/interfaces'; import { Telemetry } from '@/telemetry'; import { ConcurrencyQueue } from './concurrency-queue'; @@ -15,13 +15,15 @@ import { ConcurrencyQueue } from './concurrency-queue'; export const CLOUD_TEMP_PRODUCTION_LIMIT = 999; export const CLOUD_TEMP_REPORTABLE_THRESHOLDS = [5, 10, 20, 50, 100, 200]; +export type ConcurrencyQueueType = 'production' | 'evaluation'; + @Service() export class ConcurrencyControlService { private isEnabled: boolean; - private readonly productionLimit: number; + private readonly limits: Map; - private readonly productionQueue: ConcurrencyQueue; + private readonly queues: Map; private readonly limitsToReport = CLOUD_TEMP_REPORTABLE_THRESHOLDS.map( (t) => CLOUD_TEMP_PRODUCTION_LIMIT - t, @@ -35,52 +37,74 @@ export class ConcurrencyControlService { ) { this.logger = this.logger.scoped('concurrency'); - this.productionLimit = config.getEnv('executions.concurrency.productionLimit'); + this.limits = new Map([ + ['production', config.getEnv('executions.concurrency.productionLimit')], + ['evaluation', config.getEnv('executions.concurrency.evaluationLimit')], + ]); - if (this.productionLimit === 0) { - throw new InvalidConcurrencyLimitError(this.productionLimit); - } + this.limits.forEach((limit, type) => { + if (limit === 0) { + throw new InvalidConcurrencyLimitError(limit); + } - if (this.productionLimit < -1) { - this.productionLimit = -1; - } + if (limit < -1) { + this.limits.set(type, -1); + } + }); - if (this.productionLimit === -1 || config.getEnv('executions.mode') === 'queue') { + if ( + Array.from(this.limits.values()).every((limit) => limit === -1) || + config.getEnv('executions.mode') === 'queue' + ) { this.isEnabled = false; return; } - this.productionQueue = new ConcurrencyQueue(this.productionLimit); + this.queues = new Map(); + this.limits.forEach((limit, type) => { + if (limit > 0) { + this.queues.set(type, new ConcurrencyQueue(limit)); + } + }); this.logInit(); this.isEnabled = true; - this.productionQueue.on('concurrency-check', ({ capacity }) => { - if (this.shouldReport(capacity)) { - this.telemetry.track('User hit concurrency limit', { - threshold: CLOUD_TEMP_PRODUCTION_LIMIT - capacity, - }); - } - }); + this.queues.forEach((queue, type) => { + queue.on('concurrency-check', ({ capacity }) => { + if (this.shouldReport(capacity)) { + this.telemetry.track('User hit concurrency limit', { + threshold: CLOUD_TEMP_PRODUCTION_LIMIT - capacity, + concurrencyQueue: type, + }); + } + }); - this.productionQueue.on('execution-throttled', ({ executionId }) => { - this.logger.debug('Execution throttled', { executionId }); - this.eventService.emit('execution-throttled', { executionId }); - }); + queue.on('execution-throttled', ({ executionId }) => { + this.logger.debug('Execution throttled', { executionId, type }); + this.eventService.emit('execution-throttled', { executionId, type }); + }); - this.productionQueue.on('execution-released', async (executionId) => { - this.logger.debug('Execution released', { executionId }); + queue.on('execution-released', (executionId) => { + this.logger.debug('Execution released', { executionId, type }); + }); }); } /** - * Check whether an execution is in the production queue. + * Check whether an execution is in any of the queues. */ has(executionId: string) { if (!this.isEnabled) return false; - return this.productionQueue.getAll().has(executionId); + for (const queue of this.queues.values()) { + if (queue.has(executionId)) { + return true; + } + } + + return false; } /** @@ -89,16 +113,16 @@ export class ConcurrencyControlService { async throttle({ mode, executionId }: { mode: ExecutionMode; executionId: string }) { if (!this.isEnabled || this.isUnlimited(mode)) return; - await this.productionQueue.enqueue(executionId); + await this.getQueue(mode)?.enqueue(executionId); } /** - * Release capacity back so the next execution in the production queue can proceed. + * Release capacity back so the next execution in the queue can proceed. */ release({ mode }: { mode: ExecutionMode }) { if (!this.isEnabled || this.isUnlimited(mode)) return; - this.productionQueue.dequeue(); + this.getQueue(mode)?.dequeue(); } /** @@ -107,7 +131,7 @@ export class ConcurrencyControlService { remove({ mode, executionId }: { mode: ExecutionMode; executionId: string }) { if (!this.isEnabled || this.isUnlimited(mode)) return; - this.productionQueue.remove(executionId); + this.getQueue(mode)?.remove(executionId); } /** @@ -115,24 +139,24 @@ export class ConcurrencyControlService { * enqueued executions that have response promises, as these cannot * be re-run via `Start.runEnqueuedExecutions` during startup. */ - async removeAll(activeExecutions: { [executionId: string]: IExecutingWorkflowData }) { + async removeAll(executionIdsToCancel: string[]) { if (!this.isEnabled) return; - const enqueuedProductionIds = this.productionQueue.getAll(); + this.queues.forEach((queue) => { + const enqueuedExecutionIds = queue.getAll(); - for (const id of enqueuedProductionIds) { - this.productionQueue.remove(id); - } + for (const id of enqueuedExecutionIds) { + queue.remove(id); + } + }); - const executionIds = Object.entries(activeExecutions) - .filter(([_, execution]) => execution.status === 'new' && execution.responsePromise) - .map(([executionId, _]) => executionId); + if (executionIdsToCancel.length === 0) return; - if (executionIds.length === 0) return; + await this.executionRepository.cancelMany(executionIdsToCancel); - await this.executionRepository.cancelMany(executionIds); - - this.logger.info('Canceled enqueued executions with response promises', { executionIds }); + this.logger.info('Canceled enqueued executions with response promises', { + executionIds: executionIdsToCancel, + }); } disable() { @@ -146,15 +170,28 @@ export class ConcurrencyControlService { private logInit() { this.logger.debug('Enabled'); - this.logger.debug( - [ - 'Production execution concurrency is', - this.productionLimit === -1 ? 'unlimited' : 'limited to ' + this.productionLimit.toString(), - ].join(' '), - ); + this.limits.forEach((limit, type) => { + this.logger.debug( + [ + `${capitalize(type)} execution concurrency is`, + limit === -1 ? 'unlimited' : 'limited to ' + limit.toString(), + ].join(' '), + ); + }); } private isUnlimited(mode: ExecutionMode) { + return this.getQueue(mode) === undefined; + } + + private shouldReport(capacity: number) { + return config.getEnv('deployment.type') === 'cloud' && this.limitsToReport.includes(capacity); + } + + /** + * Get the concurrency queue based on the execution mode. + */ + private getQueue(mode: ExecutionMode) { if ( mode === 'error' || mode === 'integrated' || @@ -163,15 +200,13 @@ export class ConcurrencyControlService { mode === 'manual' || mode === 'retry' ) { - return true; + return undefined; } - if (mode === 'webhook' || mode === 'trigger') return this.productionLimit === -1; + if (mode === 'webhook' || mode === 'trigger') return this.queues.get('production'); + + if (mode === 'evaluation') return this.queues.get('evaluation'); throw new UnknownExecutionModeError(mode); } - - private shouldReport(capacity: number) { - return config.getEnv('deployment.type') === 'cloud' && this.limitsToReport.includes(capacity); - } } diff --git a/packages/cli/src/concurrency/concurrency-queue.ts b/packages/cli/src/concurrency/concurrency-queue.ts index 900018889a..eac9e478d1 100644 --- a/packages/cli/src/concurrency/concurrency-queue.ts +++ b/packages/cli/src/concurrency/concurrency-queue.ts @@ -58,6 +58,10 @@ export class ConcurrencyQueue extends TypedEmitter { return new Set(this.queue.map((item) => item.executionId)); } + has(executionId: string) { + return this.queue.some((item) => item.executionId === executionId); + } + private resolveNext() { const item = this.queue.shift(); diff --git a/packages/cli/src/config/index.ts b/packages/cli/src/config/index.ts index 8839d180ff..d11fa795db 100644 --- a/packages/cli/src/config/index.ts +++ b/packages/cli/src/config/index.ts @@ -133,3 +133,5 @@ setGlobalState({ // eslint-disable-next-line import/no-default-export export default config; + +export type Config = typeof config; diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index e5bda7d81b..6041549ec5 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -35,6 +35,12 @@ export const schema = { default: -1, env: 'N8N_CONCURRENCY_PRODUCTION_LIMIT', }, + evaluationLimit: { + doc: 'Max evaluation executions allowed to run concurrently.', + format: Number, + default: -1, + env: 'N8N_CONCURRENCY_EVALUATION_LIMIT', + }, }, // A Workflow times out and gets canceled after this time (seconds). @@ -139,13 +145,6 @@ export const schema = { doc: 'Public URL where the editor is accessible. Also used for emails sent from n8n.', }, - workflowTagsDisabled: { - format: Boolean, - default: false, - env: 'N8N_WORKFLOW_TAGS_DISABLED', - doc: 'Disable workflow tags.', - }, - userManagement: { jwtSecret: { doc: 'Set a specific JWT secret (optional - n8n can generate one)', // Generated @ start.ts @@ -190,13 +189,6 @@ export const schema = { env: 'EXTERNAL_FRONTEND_HOOKS_URLS', }, - externalHookFiles: { - doc: 'Files containing external hooks. Multiple files can be separated by colon (":")', - format: String, - default: '', - env: 'EXTERNAL_HOOK_FILES', - }, - push: { backend: { format: ['sse', 'websocket'] as const, @@ -356,15 +348,6 @@ export const schema = { }, }, - sourceControl: { - defaultKeyPairType: { - doc: 'Default SSH key type to use when generating SSH keys', - format: ['rsa', 'ed25519'] as const, - default: 'ed25519', - env: 'N8N_SOURCECONTROL_DEFAULT_SSH_KEY_TYPE', - }, - }, - workflowHistory: { enabled: { doc: 'Whether to save workflow history versions', @@ -387,13 +370,4 @@ export const schema = { env: 'N8N_PROXY_HOPS', doc: 'Number of reverse-proxies n8n is running behind', }, - - featureFlags: { - partialExecutionVersionDefault: { - format: String, - default: '0', - env: 'PARTIAL_EXECUTION_VERSION_DEFAULT', - doc: 'Set this to 1 to enable the new partial execution logic by default.', - }, - }, }; diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index abcf298d3d..6411c91bac 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -104,6 +104,7 @@ export const LICENSE_QUOTAS = { WORKFLOW_HISTORY_PRUNE_LIMIT: 'quota:workflowHistoryPrune', TEAM_PROJECT_LIMIT: 'quota:maxTeamProjects', AI_CREDITS: 'quota:aiCredits', + API_KEYS_PER_USER_LIMIT: 'quota:apiKeysPerUserLimit', } as const; export const UNLIMITED_LICENSE_QUOTA = -1; diff --git a/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts b/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts index dc40d3357d..eb13081b48 100644 --- a/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/api-keys.controller.test.ts @@ -3,8 +3,10 @@ import { mock } from 'jest-mock-extended'; import type { ApiKey } from '@/databases/entities/api-key'; import type { User } from '@/databases/entities/user'; +import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; import { EventService } from '@/events/event.service'; -import type { ApiKeysRequest, AuthenticatedRequest } from '@/requests'; +import { License } from '@/license'; +import type { AuthenticatedRequest } from '@/requests'; import { PublicApiKeyService } from '@/services/public-api-key.service'; import { mockInstance } from '@test/mocking'; @@ -13,6 +15,8 @@ import { ApiKeysController } from '../api-keys.controller'; describe('ApiKeysController', () => { const publicApiKeyService = mockInstance(PublicApiKeyService); const eventService = mockInstance(EventService); + mockInstance(ApiKeyRepository); + mockInstance(License); const controller = Container.get(ApiKeysController); let req: AuthenticatedRequest; @@ -28,7 +32,7 @@ describe('ApiKeysController', () => { id: '123', userId: '123', label: 'My API Key', - apiKey: 'apiKey********', + apiKey: 'apiKey123', createdAt: new Date(), } as ApiKey; @@ -36,14 +40,25 @@ describe('ApiKeysController', () => { publicApiKeyService.createPublicApiKeyForUser.mockResolvedValue(apiKeyData); + publicApiKeyService.redactApiKey.mockImplementation(() => '***123'); + // Act - const newApiKey = await controller.createAPIKey(req); + const newApiKey = await controller.createAPIKey(req, mock(), mock()); // Assert expect(publicApiKeyService.createPublicApiKeyForUser).toHaveBeenCalled(); - expect(apiKeyData).toEqual(newApiKey); + expect(newApiKey).toEqual( + expect.objectContaining({ + id: '123', + userId: '123', + label: 'My API Key', + apiKey: '***123', + createdAt: expect.any(Date), + rawApiKey: 'apiKey123', + }), + ); expect(eventService.emit).toHaveBeenCalledWith( 'public-api-key-created', expect.objectContaining({ user: req.user, publicApi: false }), @@ -64,7 +79,9 @@ describe('ApiKeysController', () => { updatedAt: new Date(), } as ApiKey; - publicApiKeyService.getRedactedApiKeysForUser.mockResolvedValue([apiKeyData]); + publicApiKeyService.getRedactedApiKeysForUser.mockResolvedValue([ + { ...apiKeyData, expiresAt: null }, + ]); // Act @@ -72,7 +89,7 @@ describe('ApiKeysController', () => { // Assert - expect(apiKeys).toEqual([apiKeyData]); + expect(apiKeys).toEqual([{ ...apiKeyData, expiresAt: null }]); expect(publicApiKeyService.getRedactedApiKeysForUser).toHaveBeenCalledWith( expect.objectContaining({ id: req.user.id }), ); @@ -91,11 +108,11 @@ describe('ApiKeysController', () => { mfaEnabled: false, }); - const req = mock({ user, params: { id: user.id } }); + const req = mock({ user, params: { id: user.id } }); // Act - await controller.deleteAPIKey(req); + await controller.deleteAPIKey(req, mock(), user.id); publicApiKeyService.deleteApiKeyForUser.mockResolvedValue(); diff --git a/packages/cli/src/controllers/__tests__/curl.controller.test.ts b/packages/cli/src/controllers/__tests__/curl.controller.test.ts deleted file mode 100644 index 0c23d925e4..0000000000 --- a/packages/cli/src/controllers/__tests__/curl.controller.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { Request } from 'express'; -import { mock } from 'jest-mock-extended'; - -import { CurlController } from '@/controllers/curl.controller'; -import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import type { CurlService } from '@/services/curl.service'; - -describe('CurlController', () => { - const service = mock(); - const controller = new CurlController(service); - - beforeEach(() => jest.clearAllMocks()); - - describe('toJson', () => { - it('should throw BadRequestError when invalid cURL command is provided', () => { - const req = mock(); - service.toHttpNodeParameters.mockImplementation(() => { - throw new Error(); - }); - - expect(() => controller.toJson(req)).toThrow(BadRequestError); - }); - - it('should return flattened parameters when valid cURL command is provided', () => { - const curlCommand = 'curl -v -X GET https://test.n8n.berlin/users'; - const req = mock(); - req.body = { curlCommand }; - service.toHttpNodeParameters.mockReturnValue({ - url: 'https://test.n8n.berlin/users', - authentication: 'none', - method: 'GET', - sendHeaders: false, - sendQuery: false, - options: { - redirect: { redirect: {} }, - response: { response: {} }, - }, - sendBody: false, - }); - - const result = controller.toJson(req); - expect(result).toEqual({ - 'parameters.method': 'GET', - 'parameters.url': 'https://test.n8n.berlin/users', - 'parameters.authentication': 'none', - 'parameters.sendBody': false, - 'parameters.sendHeaders': false, - 'parameters.sendQuery': false, - }); - }); - }); -}); diff --git a/packages/cli/src/controllers/api-keys.controller.ts b/packages/cli/src/controllers/api-keys.controller.ts index db53a00449..e2a824068a 100644 --- a/packages/cli/src/controllers/api-keys.controller.ts +++ b/packages/cli/src/controllers/api-keys.controller.ts @@ -1,9 +1,13 @@ -import { type RequestHandler } from 'express'; +import { CreateApiKeyRequestDto, UpdateApiKeyRequestDto } from '@n8n/api-types'; +import type { RequestHandler } from 'express'; -import { Delete, Get, Post, RestController } from '@/decorators'; +import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; +import { Body, Delete, Get, Param, Patch, Post, RestController } from '@/decorators'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { EventService } from '@/events/event.service'; +import { License } from '@/license'; import { isApiEnabled } from '@/public-api'; -import { ApiKeysRequest, AuthenticatedRequest } from '@/requests'; +import { AuthenticatedRequest } from '@/requests'; import { PublicApiKeyService } from '@/services/public-api-key.service'; export const isApiEnabledMiddleware: RequestHandler = (_, res, next) => { @@ -19,18 +23,38 @@ export class ApiKeysController { constructor( private readonly eventService: EventService, private readonly publicApiKeyService: PublicApiKeyService, + private readonly apiKeysRepository: ApiKeyRepository, + private readonly license: License, ) {} /** * Create an API Key */ @Post('/', { middlewares: [isApiEnabledMiddleware] }) - async createAPIKey(req: AuthenticatedRequest) { - const newApiKey = await this.publicApiKeyService.createPublicApiKeyForUser(req.user); + async createAPIKey( + req: AuthenticatedRequest, + _res: Response, + @Body { label, expiresAt }: CreateApiKeyRequestDto, + ) { + const currentNumberOfApiKeys = await this.apiKeysRepository.countBy({ userId: req.user.id }); + + if (currentNumberOfApiKeys >= this.license.getApiKeysPerUserLimit()) { + throw new BadRequestError('You have reached the maximum number of API keys allowed.'); + } + + const newApiKey = await this.publicApiKeyService.createPublicApiKeyForUser(req.user, { + label, + expiresAt, + }); this.eventService.emit('public-api-key-created', { user: req.user, publicApi: false }); - return newApiKey; + return { + ...newApiKey, + apiKey: this.publicApiKeyService.redactApiKey(newApiKey.apiKey), + rawApiKey: newApiKey.apiKey, + expiresAt, + }; } /** @@ -46,11 +70,28 @@ export class ApiKeysController { * Delete an API Key */ @Delete('/:id', { middlewares: [isApiEnabledMiddleware] }) - async deleteAPIKey(req: ApiKeysRequest.DeleteAPIKey) { - await this.publicApiKeyService.deleteApiKeyForUser(req.user, req.params.id); + async deleteAPIKey(req: AuthenticatedRequest, _res: Response, @Param('id') apiKeyId: string) { + await this.publicApiKeyService.deleteApiKeyForUser(req.user, apiKeyId); this.eventService.emit('public-api-key-deleted', { user: req.user, publicApi: false }); return { success: true }; } + + /** + * Patch an API Key + */ + @Patch('/:id', { middlewares: [isApiEnabledMiddleware] }) + async updateAPIKey( + req: AuthenticatedRequest, + _res: Response, + @Param('id') apiKeyId: string, + @Body { label }: UpdateApiKeyRequestDto, + ) { + await this.publicApiKeyService.updateApiKeyForUser(req.user, apiKeyId, { + label, + }); + + return { success: true }; + } } diff --git a/packages/cli/src/controllers/curl.controller.ts b/packages/cli/src/controllers/curl.controller.ts deleted file mode 100644 index e7a12b831f..0000000000 --- a/packages/cli/src/controllers/curl.controller.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Request } from 'express'; - -import { Post, RestController } from '@/decorators'; -import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { CurlService, flattenObject } from '@/services/curl.service'; - -@RestController('/curl') -export class CurlController { - constructor(private readonly curlService: CurlService) {} - - @Post('/to-json') - toJson(req: Request<{}, {}, { curlCommand: string }>) { - try { - const parameters = this.curlService.toHttpNodeParameters(req.body.curlCommand); - return flattenObject(parameters, 'parameters'); - } catch (e) { - throw new BadRequestError('Invalid cURL command'); - } - } -} diff --git a/packages/cli/src/controllers/e2e.controller.ts b/packages/cli/src/controllers/e2e.controller.ts index c846553d30..90ba208ce1 100644 --- a/packages/cli/src/controllers/e2e.controller.ts +++ b/packages/cli/src/controllers/e2e.controller.ts @@ -103,13 +103,29 @@ export class E2EController { [LICENSE_FEATURES.AI_CREDITS]: false, }; - private numericFeatures: Record = { + private static readonly numericFeaturesDefaults: Record = { [LICENSE_QUOTAS.TRIGGER_LIMIT]: -1, [LICENSE_QUOTAS.VARIABLES_LIMIT]: -1, [LICENSE_QUOTAS.USERS_LIMIT]: -1, [LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT]: -1, [LICENSE_QUOTAS.TEAM_PROJECT_LIMIT]: 0, [LICENSE_QUOTAS.AI_CREDITS]: 0, + [LICENSE_QUOTAS.API_KEYS_PER_USER_LIMIT]: 1, + }; + + private numericFeatures: Record = { + [LICENSE_QUOTAS.TRIGGER_LIMIT]: + E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.TRIGGER_LIMIT], + [LICENSE_QUOTAS.VARIABLES_LIMIT]: + E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.VARIABLES_LIMIT], + [LICENSE_QUOTAS.USERS_LIMIT]: E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.USERS_LIMIT], + [LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT]: + E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT], + [LICENSE_QUOTAS.TEAM_PROJECT_LIMIT]: + E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.TEAM_PROJECT_LIMIT], + [LICENSE_QUOTAS.AI_CREDITS]: E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.AI_CREDITS], + [LICENSE_QUOTAS.API_KEYS_PER_USER_LIMIT]: + E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.API_KEYS_PER_USER_LIMIT], }; constructor( @@ -181,6 +197,11 @@ export class E2EController { for (const feature of Object.keys(this.enabledFeatures)) { this.enabledFeatures[feature as BooleanLicenseFeature] = false; } + + for (const feature of Object.keys(this.numericFeatures)) { + this.numericFeatures[feature as NumericLicenseFeature] = + E2EController.numericFeaturesDefaults[feature as NumericLicenseFeature]; + } } private async removeActiveWorkflows() { diff --git a/packages/cli/src/controllers/oauth/__tests__/oauth1-credential.controller.test.ts b/packages/cli/src/controllers/oauth/__tests__/oauth1-credential.controller.test.ts index 570181aa78..488bcf61ff 100644 --- a/packages/cli/src/controllers/oauth/__tests__/oauth1-credential.controller.test.ts +++ b/packages/cli/src/controllers/oauth/__tests__/oauth1-credential.controller.test.ts @@ -4,6 +4,7 @@ import type { Response } from 'express'; import { mock } from 'jest-mock-extended'; import { Cipher } from 'n8n-core'; import { Logger } from 'n8n-core'; +import type { IWorkflowExecuteAdditionalData } from 'n8n-workflow'; import nock from 'nock'; import { Time } from '@/constants'; @@ -19,8 +20,11 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { ExternalHooks } from '@/external-hooks'; import type { OAuthRequest } from '@/requests'; import { SecretsHelper } from '@/secrets-helpers.ee'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import { mockInstance } from '@test/mocking'; +jest.mock('@/workflow-execute-additional-data'); + describe('OAuth1CredentialController', () => { mockInstance(Logger); mockInstance(ExternalHooks); @@ -28,6 +32,9 @@ describe('OAuth1CredentialController', () => { mockInstance(VariablesService, { getAllCached: async () => [], }); + const additionalData = mock(); + (WorkflowExecuteAdditionalData.getBase as jest.Mock).mockReturnValue(additionalData); + const cipher = mockInstance(Cipher); const credentialsHelper = mockInstance(CredentialsHelper); const credentialsRepository = mockInstance(CredentialsRepository); @@ -106,6 +113,14 @@ describe('OAuth1CredentialController', () => { }), ); expect(cipher.encrypt).toHaveBeenCalledWith({ csrfSecret }); + expect(credentialsHelper.getDecrypted).toHaveBeenCalledWith( + additionalData, + credential, + credential.type, + 'internal', + undefined, + false, + ); }); }); @@ -235,6 +250,14 @@ describe('OAuth1CredentialController', () => { }), ); expect(res.render).toHaveBeenCalledWith('oauth-callback'); + expect(credentialsHelper.getDecrypted).toHaveBeenCalledWith( + additionalData, + credential, + credential.type, + 'internal', + undefined, + true, + ); }); }); }); diff --git a/packages/cli/src/controllers/oauth/__tests__/oauth2-credential.controller.test.ts b/packages/cli/src/controllers/oauth/__tests__/oauth2-credential.controller.test.ts index 53bba08c58..d075dcd198 100644 --- a/packages/cli/src/controllers/oauth/__tests__/oauth2-credential.controller.test.ts +++ b/packages/cli/src/controllers/oauth/__tests__/oauth2-credential.controller.test.ts @@ -4,9 +4,10 @@ import { type Response } from 'express'; import { mock } from 'jest-mock-extended'; import { Cipher } from 'n8n-core'; import { Logger } from 'n8n-core'; +import type { IWorkflowExecuteAdditionalData } from 'n8n-workflow'; import nock from 'nock'; -import { Time } from '@/constants'; +import { CREDENTIAL_BLANKING_VALUE, Time } from '@/constants'; import { OAuth2CredentialController } from '@/controllers/oauth/oauth2-credential.controller'; import { CredentialsHelper } from '@/credentials-helper'; import type { CredentialsEntity } from '@/databases/entities/credentials-entity'; @@ -19,14 +20,20 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { ExternalHooks } from '@/external-hooks'; import type { OAuthRequest } from '@/requests'; import { SecretsHelper } from '@/secrets-helpers.ee'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import { mockInstance } from '@test/mocking'; +jest.mock('@/workflow-execute-additional-data'); + describe('OAuth2CredentialController', () => { mockInstance(Logger); mockInstance(SecretsHelper); mockInstance(VariablesService, { getAllCached: async () => [], }); + const additionalData = mock(); + (WorkflowExecuteAdditionalData.getBase as jest.Mock).mockReturnValue(additionalData); + const cipher = mockInstance(Cipher); const externalHooks = mockInstance(ExternalHooks); const credentialsHelper = mockInstance(CredentialsHelper); @@ -108,6 +115,14 @@ describe('OAuth2CredentialController', () => { type: 'oAuth2Api', }), ); + expect(credentialsHelper.getDecrypted).toHaveBeenCalledWith( + additionalData, + credential, + credential.type, + 'internal', + undefined, + false, + ); }); }); @@ -237,6 +252,94 @@ describe('OAuth2CredentialController', () => { await controller.handleCallback(req, res); + expect(externalHooks.run).toHaveBeenCalledWith('oauth2.callback', [ + expect.objectContaining({ + clientId: 'test-client-id', + redirectUri: 'http://localhost:5678/rest/oauth2-credential/callback', + }), + ]); + expect(cipher.encrypt).toHaveBeenCalledWith({ + oauthTokenData: { access_token: 'access-token', refresh_token: 'refresh-token' }, + }); + expect(credentialsRepository.update).toHaveBeenCalledWith( + '1', + expect.objectContaining({ + data: 'encrypted', + id: '1', + name: 'Test Credential', + type: 'oAuth2Api', + }), + ); + expect(res.render).toHaveBeenCalledWith('oauth-callback'); + expect(credentialsHelper.getDecrypted).toHaveBeenCalledWith( + additionalData, + credential, + credential.type, + 'internal', + undefined, + true, + ); + }); + + it('merges oauthTokenData if it already exists', async () => { + credentialsRepository.findOneBy.mockResolvedValueOnce(credential); + credentialsHelper.getDecrypted.mockResolvedValueOnce({ + csrfSecret, + oauthTokenData: { token: true }, + }); + jest.spyOn(Csrf.prototype, 'verify').mockReturnValueOnce(true); + nock('https://example.domain') + .post( + '/token', + 'code=code&grant_type=authorization_code&redirect_uri=http%3A%2F%2Flocalhost%3A5678%2Frest%2Foauth2-credential%2Fcallback', + ) + .reply(200, { access_token: 'access-token', refresh_token: 'refresh-token' }); + cipher.encrypt.mockReturnValue('encrypted'); + + await controller.handleCallback(req, res); + + expect(externalHooks.run).toHaveBeenCalledWith('oauth2.callback', [ + expect.objectContaining({ + clientId: 'test-client-id', + redirectUri: 'http://localhost:5678/rest/oauth2-credential/callback', + }), + ]); + expect(cipher.encrypt).toHaveBeenCalledWith({ + oauthTokenData: { + token: true, + access_token: 'access-token', + refresh_token: 'refresh-token', + }, + }); + expect(credentialsRepository.update).toHaveBeenCalledWith( + '1', + expect.objectContaining({ + data: 'encrypted', + id: '1', + name: 'Test Credential', + type: 'oAuth2Api', + }), + ); + expect(res.render).toHaveBeenCalledWith('oauth-callback'); + }); + + it('overwrites oauthTokenData if it is a string', async () => { + credentialsRepository.findOneBy.mockResolvedValueOnce(credential); + credentialsHelper.getDecrypted.mockResolvedValueOnce({ + csrfSecret, + oauthTokenData: CREDENTIAL_BLANKING_VALUE, + }); + jest.spyOn(Csrf.prototype, 'verify').mockReturnValueOnce(true); + nock('https://example.domain') + .post( + '/token', + 'code=code&grant_type=authorization_code&redirect_uri=http%3A%2F%2Flocalhost%3A5678%2Frest%2Foauth2-credential%2Fcallback', + ) + .reply(200, { access_token: 'access-token', refresh_token: 'refresh-token' }); + cipher.encrypt.mockReturnValue('encrypted'); + + await controller.handleCallback(req, res); + expect(externalHooks.run).toHaveBeenCalledWith('oauth2.callback', [ expect.objectContaining({ clientId: 'test-client-id', diff --git a/packages/cli/src/controllers/oauth/abstract-oauth.controller.ts b/packages/cli/src/controllers/oauth/abstract-oauth.controller.ts index bac924a023..5098cec28d 100644 --- a/packages/cli/src/controllers/oauth/abstract-oauth.controller.ts +++ b/packages/cli/src/controllers/oauth/abstract-oauth.controller.ts @@ -86,9 +86,30 @@ export abstract class AbstractOAuthController { return await WorkflowExecuteAdditionalData.getBase(); } - protected async getDecryptedData( + /** + * Allow decrypted data to evaluate expressions that include $secrets and apply overwrites + */ + protected async getDecryptedDataForAuthUri( credential: ICredentialsDb, additionalData: IWorkflowExecuteAdditionalData, + ) { + return await this.getDecryptedData(credential, additionalData, false); + } + + /** + * Do not apply overwrites here because that removes the CSRF state, and breaks the oauth flow + */ + protected async getDecryptedDataForCallback( + credential: ICredentialsDb, + additionalData: IWorkflowExecuteAdditionalData, + ) { + return await this.getDecryptedData(credential, additionalData, true); + } + + private async getDecryptedData( + credential: ICredentialsDb, + additionalData: IWorkflowExecuteAdditionalData, + raw: boolean, ) { return await this.credentialsHelper.getDecrypted( additionalData, @@ -96,7 +117,7 @@ export abstract class AbstractOAuthController { credential.type, 'internal', undefined, - true, + raw, ); } @@ -183,7 +204,10 @@ export abstract class AbstractOAuthController { } const additionalData = await this.getAdditionalData(); - const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); + const decryptedDataOriginal = await this.getDecryptedDataForCallback( + credential, + additionalData, + ); const oauthCredentials = this.applyDefaultsAndOverwrites( credential, decryptedDataOriginal, diff --git a/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts b/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts index 0211c463a9..fe9d22edd8 100644 --- a/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts @@ -35,7 +35,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { async getAuthUri(req: OAuthRequest.OAuth1Credential.Auth): Promise { const credential = await this.getCredential(req); const additionalData = await this.getAdditionalData(); - const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); + const decryptedDataOriginal = await this.getDecryptedDataForAuthUri(credential, additionalData); const oauthCredentials = this.applyDefaultsAndOverwrites( credential, decryptedDataOriginal, diff --git a/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts b/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts index 0f563993ff..b9ed0d1126 100644 --- a/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts @@ -23,7 +23,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { async getAuthUri(req: OAuthRequest.OAuth2Credential.Auth): Promise { const credential = await this.getCredential(req); const additionalData = await this.getAdditionalData(); - const decryptedDataOriginal = await this.getDecryptedData(credential, additionalData); + const decryptedDataOriginal = await this.getDecryptedDataForAuthUri(credential, additionalData); // At some point in the past we saved hidden scopes to credentials (but shouldn't) // Delete scope before applying defaults to make sure new scopes are present on reconnect @@ -133,7 +133,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { set(oauthToken.data, 'callbackQueryString', omit(req.query, 'state', 'code')); } - if (decryptedDataOriginal.oauthTokenData) { + if (typeof decryptedDataOriginal.oauthTokenData === 'object') { // Only overwrite supplied data as some providers do for example just return the // refresh_token on the very first request and not on subsequent ones. Object.assign(decryptedDataOriginal.oauthTokenData, oauthToken.data); diff --git a/packages/cli/src/controllers/project.controller.ts b/packages/cli/src/controllers/project.controller.ts index ab48f38c5b..665eba9636 100644 --- a/packages/cli/src/controllers/project.controller.ts +++ b/packages/cli/src/controllers/project.controller.ts @@ -1,7 +1,9 @@ +import { CreateProjectDto, DeleteProjectDto, UpdateProjectDto } from '@n8n/api-types'; import { combineScopes } from '@n8n/permissions'; import type { Scope } from '@n8n/permissions'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In, Not } from '@n8n/typeorm'; +import { Response } from 'express'; import type { Project } from '@/databases/entities/project'; import { ProjectRepository } from '@/databases/repositories/project.repository'; @@ -14,11 +16,15 @@ import { Patch, ProjectScope, Delete, + Body, + Param, + Query, } from '@/decorators'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { EventService } from '@/events/event.service'; -import { ProjectRequest } from '@/requests'; +import type { ProjectRequest } from '@/requests'; +import { AuthenticatedRequest } from '@/requests'; import { ProjectService, TeamProjectOverQuotaError, @@ -36,7 +42,7 @@ export class ProjectController { ) {} @Get('/') - async getAllProjects(req: ProjectRequest.GetAll): Promise { + async getAllProjects(req: AuthenticatedRequest): Promise { return await this.projectsService.getAccessibleProjects(req.user); } @@ -49,14 +55,9 @@ export class ProjectController { @GlobalScope('project:create') // Using admin as all plans that contain projects should allow admins at the very least @Licensed('feat:projectRole:admin') - async createProject(req: ProjectRequest.Create) { + async createProject(req: AuthenticatedRequest, _res: Response, @Body payload: CreateProjectDto) { try { - const project = await this.projectsService.createTeamProject( - req.body.name, - req.user, - undefined, - req.body.icon, - ); + const project = await this.projectsService.createTeamProject(req.user, payload); this.eventService.emit('team-project-created', { userId: req.user.id, @@ -83,7 +84,8 @@ export class ProjectController { @Get('/my-projects') async getMyProjects( - req: ProjectRequest.GetMyProjects, + req: AuthenticatedRequest, + _res: Response, ): Promise { const relations = await this.projectsService.getProjectRelationsForUser(req.user); const otherTeamProject = req.user.hasGlobalScope('project:read') @@ -98,10 +100,7 @@ export class ProjectController { for (const pr of relations) { const result: ProjectRequest.GetMyProjectsResponse[number] = Object.assign( this.projectRepository.create(pr.project), - { - role: pr.role, - scopes: req.query.includeScopes ? ([] as Scope[]) : undefined, - }, + { role: pr.role, scopes: [] }, ); if (result.scopes) { @@ -124,7 +123,7 @@ export class ProjectController { // own this relationship in that case we use the global user role // instead of the relation role, which is for another user. role: req.user.role, - scopes: req.query.includeScopes ? [] : undefined, + scopes: [], }, ); @@ -148,7 +147,7 @@ export class ProjectController { } @Get('/personal') - async getPersonalProject(req: ProjectRequest.GetPersonalProject) { + async getPersonalProject(req: AuthenticatedRequest) { const project = await this.projectsService.getPersonalProject(req.user); if (!project) { throw new NotFoundError('Could not find a personal project for this user'); @@ -167,10 +166,14 @@ export class ProjectController { @Get('/:projectId') @ProjectScope('project:read') - async getProject(req: ProjectRequest.Get): Promise { + async getProject( + req: AuthenticatedRequest, + _res: Response, + @Param('projectId') projectId: string, + ): Promise { const [{ id, name, icon, type }, relations] = await Promise.all([ - this.projectsService.getProject(req.params.projectId), - this.projectsService.getProjectRelations(req.params.projectId), + this.projectsService.getProject(projectId), + this.projectsService.getProjectRelations(projectId), ]); const myRelation = relations.find((r) => r.userId === req.user.id); @@ -197,13 +200,19 @@ export class ProjectController { @Patch('/:projectId') @ProjectScope('project:update') - async updateProject(req: ProjectRequest.Update) { - if (req.body.name) { - await this.projectsService.updateProject(req.body.name, req.params.projectId, req.body.icon); + async updateProject( + req: AuthenticatedRequest, + _res: Response, + @Body payload: UpdateProjectDto, + @Param('projectId') projectId: string, + ) { + const { name, icon, relations } = payload; + if (name || icon) { + await this.projectsService.updateProject(projectId, { name, icon }); } - if (req.body.relations) { + if (relations) { try { - await this.projectsService.syncProjectRelations(req.params.projectId, req.body.relations); + await this.projectsService.syncProjectRelations(projectId, relations); } catch (e) { if (e instanceof UnlicensedProjectRoleError) { throw new BadRequestError(e.message); @@ -214,25 +223,30 @@ export class ProjectController { this.eventService.emit('team-project-updated', { userId: req.user.id, role: req.user.role, - members: req.body.relations, - projectId: req.params.projectId, + members: relations, + projectId, }); } } @Delete('/:projectId') @ProjectScope('project:delete') - async deleteProject(req: ProjectRequest.Delete) { - await this.projectsService.deleteProject(req.user, req.params.projectId, { - migrateToProject: req.query.transferId, + async deleteProject( + req: AuthenticatedRequest, + _res: Response, + @Query query: DeleteProjectDto, + @Param('projectId') projectId: string, + ) { + await this.projectsService.deleteProject(req.user, projectId, { + migrateToProject: query.transferId, }); this.eventService.emit('team-project-deleted', { userId: req.user.id, role: req.user.role, - projectId: req.params.projectId, - removalType: req.query.transferId !== undefined ? 'transfer' : 'delete', - targetProjectId: req.query.transferId, + projectId, + removalType: query.transferId !== undefined ? 'transfer' : 'delete', + targetProjectId: query.transferId, }); } } diff --git a/packages/cli/src/controllers/tags.controller.ts b/packages/cli/src/controllers/tags.controller.ts index a6551a021d..52432d2d4b 100644 --- a/packages/cli/src/controllers/tags.controller.ts +++ b/packages/cli/src/controllers/tags.controller.ts @@ -1,54 +1,60 @@ -import { Request, Response, NextFunction } from 'express'; +import { CreateOrUpdateTagRequestDto, RetrieveTagQueryDto } from '@n8n/api-types'; +import { Response } from 'express'; -import config from '@/config'; -import { Delete, Get, Middleware, Patch, Post, RestController, GlobalScope } from '@/decorators'; -import { BadRequestError } from '@/errors/response-errors/bad-request.error'; -import { TagsRequest } from '@/requests'; +import { + Delete, + Get, + Patch, + Post, + RestController, + GlobalScope, + Body, + Param, + Query, +} from '@/decorators'; +import { AuthenticatedRequest } from '@/requests'; import { TagService } from '@/services/tag.service'; @RestController('/tags') export class TagsController { - private config = config; - constructor(private readonly tagService: TagService) {} - // TODO: move this into a new decorator `@IfEnabled('workflowTagsDisabled')` - @Middleware() - workflowsEnabledMiddleware(_req: Request, _res: Response, next: NextFunction) { - if (this.config.getEnv('workflowTagsDisabled')) - throw new BadRequestError('Workflow tags are disabled'); - next(); - } - @Get('/') @GlobalScope('tag:list') - async getAll(req: TagsRequest.GetAll) { - return await this.tagService.getAll({ withUsageCount: req.query.withUsageCount === 'true' }); + async getAll(_req: AuthenticatedRequest, _res: Response, @Query query: RetrieveTagQueryDto) { + return await this.tagService.getAll({ withUsageCount: query.withUsageCount }); } @Post('/') @GlobalScope('tag:create') - async createTag(req: TagsRequest.Create) { - const tag = this.tagService.toEntity({ name: req.body.name }); + async createTag( + _req: AuthenticatedRequest, + _res: Response, + @Body payload: CreateOrUpdateTagRequestDto, + ) { + const { name } = payload; + const tag = this.tagService.toEntity({ name }); return await this.tagService.save(tag, 'create'); } @Patch('/:id(\\w+)') @GlobalScope('tag:update') - async updateTag(req: TagsRequest.Update) { - const newTag = this.tagService.toEntity({ id: req.params.id, name: req.body.name.trim() }); + async updateTag( + _req: AuthenticatedRequest, + _res: Response, + @Param('id') tagId: string, + @Body payload: CreateOrUpdateTagRequestDto, + ) { + const newTag = this.tagService.toEntity({ id: tagId, name: payload.name }); return await this.tagService.save(newTag, 'update'); } @Delete('/:id(\\w+)') @GlobalScope('tag:delete') - async deleteTag(req: TagsRequest.Delete) { - const { id } = req.params; - - await this.tagService.delete(id); - + async deleteTag(_req: AuthenticatedRequest, _res: Response, @Param('id') tagId: string) { + await this.tagService.delete(tagId); return true; } } diff --git a/packages/cli/src/controllers/users.controller.ts b/packages/cli/src/controllers/users.controller.ts index 3177c2c23b..80d38fcfcd 100644 --- a/packages/cli/src/controllers/users.controller.ts +++ b/packages/cli/src/controllers/users.controller.ts @@ -240,7 +240,7 @@ export class UsersController { } for (const credential of ownedCredentials) { - await this.credentialsService.delete(credential); + await this.credentialsService.delete(userToDelete, credential.id); } await this.userService.getManager().transaction(async (trx) => { diff --git a/packages/cli/src/credentials/__tests__/credentials.controller.test.ts b/packages/cli/src/credentials/__tests__/credentials.controller.test.ts index 13e72e8003..68d2f26750 100644 --- a/packages/cli/src/credentials/__tests__/credentials.controller.test.ts +++ b/packages/cli/src/credentials/__tests__/credentials.controller.test.ts @@ -33,7 +33,7 @@ describe('CredentialsController', () => { }); describe('createCredentials', () => { - it('it should create new credentials and emit "credentials-created"', async () => { + it('should create new credentials and emit "credentials-created"', async () => { // Arrange const newCredentialsPayload = createNewCredentialsPayload(); diff --git a/packages/cli/src/credentials/credentials.controller.ts b/packages/cli/src/credentials/credentials.controller.ts index 4cc0b500f2..6b4fd8472a 100644 --- a/packages/cli/src/credentials/credentials.controller.ts +++ b/packages/cli/src/credentials/credentials.controller.ts @@ -198,7 +198,7 @@ export class CredentialsController { throw new BadRequestError('Managed credentials cannot be updated'); } - const decryptedData = this.credentialsService.decrypt(credential); + const decryptedData = this.credentialsService.decrypt(credential, true); const preparedCredentialData = await this.credentialsService.prepareUpdateData( req.body, decryptedData, @@ -251,7 +251,7 @@ export class CredentialsController { ); } - await this.credentialsService.delete(credential); + await this.credentialsService.delete(req.user, credential.id); this.eventService.emit('credentials-deleted', { user: req.user, diff --git a/packages/cli/src/credentials/credentials.service.ts b/packages/cli/src/credentials/credentials.service.ts index e0939beccb..18d01a198a 100644 --- a/packages/cli/src/credentials/credentials.service.ts +++ b/packages/cli/src/credentials/credentials.service.ts @@ -20,7 +20,6 @@ import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; import { CredentialTypes } from '@/credential-types'; import { createCredentialsFromCredentialsEntity } from '@/credentials-helper'; import { CredentialsEntity } from '@/databases/entities/credentials-entity'; -import type { ProjectRelation } from '@/databases/entities/project-relation'; import { SharedCredentials } from '@/databases/entities/shared-credentials'; import type { User } from '@/databases/entities/user'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; @@ -85,23 +84,6 @@ export class CredentialsService { listQueryOptions.includeData = true; } - let projectRelations: ProjectRelation[] | undefined = undefined; - if (includeScopes) { - projectRelations = await this.projectService.getProjectRelationsForUser(user); - if (listQueryOptions.filter?.projectId && user.hasGlobalScope('credential:list')) { - // Only instance owners and admins have the credential:list scope - // Those users should be able to use _all_ credentials within their workflows. - // TODO: Change this so we filter by `workflowId` in this case. Require a slight FE change - const projectRelation = projectRelations.find( - (relation) => relation.projectId === listQueryOptions.filter?.projectId, - ); - if (projectRelation?.role === 'project:personalOwner') { - // Will not affect team projects as these have admins, not owners. - delete listQueryOptions.filter?.projectId; - } - } - } - if (returnAll) { let credentials = await this.credentialsRepository.findMany(listQueryOptions); @@ -123,9 +105,8 @@ export class CredentialsService { } if (includeScopes) { - credentials = credentials.map((c) => - this.roleService.addScopes(c, user, projectRelations!), - ); + const projectRelations = await this.projectService.getProjectRelationsForUser(user); + credentials = credentials.map((c) => this.roleService.addScopes(c, user, projectRelations)); } if (includeData) { @@ -179,7 +160,8 @@ export class CredentialsService { } if (includeScopes) { - credentials = credentials.map((c) => this.roleService.addScopes(c, user, projectRelations!)); + const projectRelations = await this.projectService.getProjectRelationsForUser(user); + credentials = credentials.map((c) => this.roleService.addScopes(c, user, projectRelations)); } if (includeData) { @@ -424,10 +406,26 @@ export class CredentialsService { return result; } - async delete(credentials: CredentialsEntity) { - await this.externalHooks.run('credentials.delete', [credentials.id]); + /** + * Deletes a credential. + * + * If the user does not have permission to delete the credential this does + * nothing and returns void. + */ + async delete(user: User, credentialId: string) { + await this.externalHooks.run('credentials.delete', [credentialId]); - await this.credentialsRepository.remove(credentials); + const credential = await this.sharedCredentialsRepository.findCredentialForUser( + credentialId, + user, + ['credential:delete'], + ); + + if (!credential) { + return; + } + + await this.credentialsRepository.remove(credential); } async test(user: User, credentials: ICredentialsDecrypted) { diff --git a/packages/cli/src/curlconverter.d.ts b/packages/cli/src/curlconverter.d.ts deleted file mode 100644 index 8808f9b5c7..0000000000 --- a/packages/cli/src/curlconverter.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'curlconverter' { - export function toJsonString(data: string): string; -} diff --git a/packages/cli/src/databases/entities/index.ts b/packages/cli/src/databases/entities/index.ts index e6a1dedb3f..1393b6a305 100644 --- a/packages/cli/src/databases/entities/index.ts +++ b/packages/cli/src/databases/entities/index.ts @@ -20,6 +20,7 @@ import { Settings } from './settings'; import { SharedCredentials } from './shared-credentials'; import { SharedWorkflow } from './shared-workflow'; import { TagEntity } from './tag-entity'; +import { TestCaseExecution } from './test-case-execution.ee'; import { TestDefinition } from './test-definition.ee'; import { TestMetric } from './test-metric.ee'; import { TestRun } from './test-run.ee'; @@ -64,4 +65,5 @@ export const entities = { TestDefinition, TestMetric, TestRun, + TestCaseExecution, }; diff --git a/packages/cli/src/databases/entities/project-relation.ts b/packages/cli/src/databases/entities/project-relation.ts index 736ef2b223..eb73b62502 100644 --- a/packages/cli/src/databases/entities/project-relation.ts +++ b/packages/cli/src/databases/entities/project-relation.ts @@ -1,19 +1,13 @@ +import { ProjectRole } from '@n8n/api-types'; import { Column, Entity, ManyToOne, PrimaryColumn } from '@n8n/typeorm'; import { WithTimestamps } from './abstract-entity'; import { Project } from './project'; import { User } from './user'; -// personalOwner is only used for personal projects -export type ProjectRole = - | 'project:personalOwner' - | 'project:admin' - | 'project:editor' - | 'project:viewer'; - @Entity() export class ProjectRelation extends WithTimestamps { - @Column() + @Column({ type: 'varchar' }) role: ProjectRole; @ManyToOne('User', 'projectRelations') diff --git a/packages/cli/src/databases/entities/project.ts b/packages/cli/src/databases/entities/project.ts index aa867807fd..48f86fe0e9 100644 --- a/packages/cli/src/databases/entities/project.ts +++ b/packages/cli/src/databases/entities/project.ts @@ -1,3 +1,4 @@ +import { ProjectIcon, ProjectType } from '@n8n/api-types'; import { Column, Entity, OneToMany } from '@n8n/typeorm'; import { WithTimestampsAndStringId } from './abstract-entity'; @@ -5,15 +6,12 @@ import type { ProjectRelation } from './project-relation'; import type { SharedCredentials } from './shared-credentials'; import type { SharedWorkflow } from './shared-workflow'; -export type ProjectType = 'personal' | 'team'; -export type ProjectIcon = { type: 'emoji' | 'icon'; value: string } | null; - @Entity() export class Project extends WithTimestampsAndStringId { @Column({ length: 255 }) name: string; - @Column({ length: 36 }) + @Column({ type: 'varchar', length: 36 }) type: ProjectType; @Column({ type: 'json', nullable: true }) diff --git a/packages/cli/src/databases/entities/test-case-execution.ee.ts b/packages/cli/src/databases/entities/test-case-execution.ee.ts new file mode 100644 index 0000000000..dc15ae63b5 --- /dev/null +++ b/packages/cli/src/databases/entities/test-case-execution.ee.ts @@ -0,0 +1,68 @@ +import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm'; + +import { + datetimeColumnType, + jsonColumnType, + WithStringId, +} from '@/databases/entities/abstract-entity'; +import type { ExecutionEntity } from '@/databases/entities/execution-entity'; +import { TestRun } from '@/databases/entities/test-run.ee'; + +export type TestCaseRunMetrics = Record; + +/** + * This entity represents the linking between the test runs and individual executions. + * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions + * Entries in this table are meant to outlive the execution entities, which might be pruned over time. + * This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted. + */ +@Entity({ name: 'test_case_execution' }) +export class TestCaseExecution extends WithStringId { + @ManyToOne('TestRun') + testRun: TestRun; + + @ManyToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + pastExecution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + pastExecutionId: string | null; + + @OneToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + execution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + executionId: string | null; + + @OneToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + evaluationExecution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + evaluationExecutionId: string | null; + + @Column() + status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled'; + + @Column({ type: datetimeColumnType, nullable: true }) + runAt: Date | null; + + @Column({ type: datetimeColumnType, nullable: true }) + completedAt: Date | null; + + @Column('varchar', { nullable: true }) + errorCode: string | null; + + @Column(jsonColumnType, { nullable: true }) + errorDetails: Record; + + @Column(jsonColumnType, { nullable: true }) + metrics: TestCaseRunMetrics; +} diff --git a/packages/cli/src/databases/entities/test-run.ee.ts b/packages/cli/src/databases/entities/test-run.ee.ts index 39d8e16ddd..79c7bc9f07 100644 --- a/packages/cli/src/databases/entities/test-run.ee.ts +++ b/packages/cli/src/databases/entities/test-run.ee.ts @@ -7,7 +7,7 @@ import { } from '@/databases/entities/abstract-entity'; import { TestDefinition } from '@/databases/entities/test-definition.ee'; -type TestRunStatus = 'new' | 'running' | 'completed' | 'error'; +type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled'; export type AggregatedTestRunMetrics = Record; diff --git a/packages/cli/src/databases/migrations/common/1714133768519-CreateProject.ts b/packages/cli/src/databases/migrations/common/1714133768519-CreateProject.ts index fe101fac7a..d7b8a2d47c 100644 --- a/packages/cli/src/databases/migrations/common/1714133768519-CreateProject.ts +++ b/packages/cli/src/databases/migrations/common/1714133768519-CreateProject.ts @@ -1,7 +1,7 @@ +import type { ProjectRole } from '@n8n/api-types'; import { ApplicationError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { User } from '@/databases/entities/user'; import type { MigrationContext, ReversibleMigration } from '@/databases/types'; import { generateNanoId } from '@/databases/utils/generators'; diff --git a/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts b/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts new file mode 100644 index 0000000000..f7e840cb49 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts @@ -0,0 +1,47 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const testCaseExecutionTableName = 'test_case_execution'; + +export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column } }: MigrationContext) { + await createTable(testCaseExecutionTableName) + .withColumns( + column('id').varchar(36).primary.notNull, + column('testRunId').varchar(36).notNull, + column('pastExecutionId').int, // Might be null if execution was deleted after the test run + column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run + column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled + column('status').varchar().notNull, + column('runAt').timestamp(), + column('completedAt').timestamp(), + column('errorCode').varchar(), + column('errorDetails').json, + column('metrics').json, + ) + .withIndexOn('testRunId') + .withForeignKey('testRunId', { + tableName: 'test_run', + columnName: 'id', + onDelete: 'CASCADE', + }) + .withForeignKey('pastExecutionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }) + .withForeignKey('executionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }) + .withForeignKey('evaluationExecutionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }).withTimestamps; + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable(testCaseExecutionTableName); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index b76409c0c1..32b1dd9751 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 7cf90bde5b..c5547271b7 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 363a6e47c3..b8b8e26d3d 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts b/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts index 8e36f0189b..a6195b4f67 100644 --- a/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts +++ b/packages/cli/src/databases/repositories/__tests__/execution.repository.test.ts @@ -1,13 +1,16 @@ import { GlobalConfig } from '@n8n/config'; +import type { SqliteConfig } from '@n8n/config/src/configs/database.config'; import { Container } from '@n8n/di'; import type { SelectQueryBuilder } from '@n8n/typeorm'; import { Not, LessThanOrEqual } from '@n8n/typeorm'; import { mock } from 'jest-mock-extended'; import { BinaryDataService } from 'n8n-core'; +import type { IRunExecutionData, IWorkflowBase } from 'n8n-workflow'; import { nanoid } from 'nanoid'; import { ExecutionEntity } from '@/databases/entities/execution-entity'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { IExecutionResponse } from '@/interfaces'; import { mockInstance, mockEntityManager } from '@test/mocking'; describe('ExecutionRepository', () => { @@ -68,4 +71,39 @@ describe('ExecutionRepository', () => { expect(binaryDataService.deleteMany).toHaveBeenCalledWith([{ executionId: '1', workflowId }]); }); }); + + describe('updateExistingExecution', () => { + test.each(['sqlite', 'postgresdb', 'mysqldb'] as const)( + 'should update execution and data in transaction on %s', + async (dbType) => { + globalConfig.database.type = dbType; + globalConfig.database.sqlite = mock({ poolSize: 1 }); + + const executionId = '1'; + const execution = mock({ + id: executionId, + data: mock(), + workflowData: mock(), + status: 'success', + }); + + const txCallback = jest.fn(); + entityManager.transaction.mockImplementation(async (cb) => { + // @ts-expect-error Mock + await cb(entityManager); + txCallback(); + }); + + await executionRepository.updateExistingExecution(executionId, execution); + + expect(entityManager.transaction).toHaveBeenCalled(); + expect(entityManager.update).toHaveBeenCalledWith( + ExecutionEntity, + { id: executionId }, + expect.objectContaining({ status: 'success' }), + ); + expect(txCallback).toHaveBeenCalledTimes(1); + }, + ); + }); }); diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 160b7ace87..9c24cea5c2 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -45,7 +45,7 @@ import type { import { separate } from '@/utils'; import { ExecutionDataRepository } from './execution-data.repository'; -import type { ExecutionData } from '../entities/execution-data'; +import { ExecutionData } from '../entities/execution-data'; import { ExecutionEntity } from '../entities/execution-entity'; import { ExecutionMetadata } from '../entities/execution-metadata'; import { SharedWorkflow } from '../entities/shared-workflow'; @@ -287,6 +287,15 @@ export class ExecutionRepository extends Repository { const { executionData, metadata, annotation, ...rest } = execution; const serializedAnnotation = this.serializeAnnotation(annotation); + if (execution.status === 'success' && executionData?.data === '[]') { + this.errorReporter.error('Found successful execution where data is empty stringified array', { + extra: { + executionId: execution.id, + workflowId: executionData?.workflowData.id, + }, + }); + } + return { ...rest, ...(options?.includeData && { @@ -378,21 +387,42 @@ export class ExecutionRepository extends Repository { customData, ...executionInformation } = execution; - if (Object.keys(executionInformation).length > 0) { - await this.update({ id: executionId }, executionInformation); + + const executionData: Partial = {}; + + if (workflowData) executionData.workflowData = workflowData; + if (data) executionData.data = stringify(data); + + const { type: dbType, sqlite: sqliteConfig } = this.globalConfig.database; + + if (dbType === 'sqlite' && sqliteConfig.poolSize === 0) { + // TODO: Delete this block of code once the sqlite legacy (non-pooling) driver is dropped. + // In the non-pooling sqlite driver we can't use transactions, because that creates nested transactions under highly concurrent loads, leading to errors in the database + + if (Object.keys(executionInformation).length > 0) { + await this.update({ id: executionId }, executionInformation); + } + + if (Object.keys(executionData).length > 0) { + // @ts-expect-error Fix typing + await this.executionDataRepository.update({ executionId }, executionData); + } + + return; } - if (data || workflowData) { - const executionData: Partial = {}; - if (workflowData) { - executionData.workflowData = workflowData; + // All other database drivers should update executions and execution-data atomically + + await this.manager.transaction(async (tx) => { + if (Object.keys(executionInformation).length > 0) { + await tx.update(ExecutionEntity, { id: executionId }, executionInformation); } - if (data) { - executionData.data = stringify(data); + + if (Object.keys(executionData).length > 0) { + // @ts-expect-error Fix typing + await tx.update(ExecutionData, { executionId }, executionData); } - // @ts-ignore - await this.executionDataRepository.update({ executionId }, executionData); - } + }); } async deleteExecutionsByFilter( diff --git a/packages/cli/src/databases/repositories/project-relation.repository.ts b/packages/cli/src/databases/repositories/project-relation.repository.ts index 75aaed76df..89e5028b19 100644 --- a/packages/cli/src/databases/repositories/project-relation.repository.ts +++ b/packages/cli/src/databases/repositories/project-relation.repository.ts @@ -1,7 +1,8 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Service } from '@n8n/di'; import { DataSource, In, Repository } from '@n8n/typeorm'; -import { ProjectRelation, type ProjectRole } from '../entities/project-relation'; +import { ProjectRelation } from '../entities/project-relation'; @Service() export class ProjectRelationRepository extends Repository { diff --git a/packages/cli/src/databases/repositories/shared-credentials.repository.ts b/packages/cli/src/databases/repositories/shared-credentials.repository.ts index d7e074595c..f696a3bd59 100644 --- a/packages/cli/src/databases/repositories/shared-credentials.repository.ts +++ b/packages/cli/src/databases/repositories/shared-credentials.repository.ts @@ -1,3 +1,4 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Service } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; import type { EntityManager, FindOptionsRelations, FindOptionsWhere } from '@n8n/typeorm'; @@ -6,7 +7,6 @@ import { DataSource, In, Not, Repository } from '@n8n/typeorm'; import { RoleService } from '@/services/role.service'; import type { Project } from '../entities/project'; -import type { ProjectRole } from '../entities/project-relation'; import { type CredentialSharingRole, SharedCredentials } from '../entities/shared-credentials'; import type { User } from '../entities/user'; diff --git a/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts b/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts new file mode 100644 index 0000000000..c9798c8941 --- /dev/null +++ b/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts @@ -0,0 +1,99 @@ +import { Service } from '@n8n/di'; +import type { EntityManager } from '@n8n/typeorm'; +import { DataSource, In, Not, Repository } from '@n8n/typeorm'; +import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial'; + +import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee'; + +@Service() +export class TestCaseExecutionRepository extends Repository { + constructor(dataSource: DataSource) { + super(TestCaseExecution, dataSource.manager); + } + + async createBatch(testRunId: string, pastExecutionIds: string[]) { + const mappings = this.create( + pastExecutionIds.map>((id) => ({ + testRun: { + id: testRunId, + }, + pastExecution: { + id, + }, + status: 'new', + })), + ); + + return await this.save(mappings); + } + + async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) { + return await this.update( + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'running', + executionId, + runAt: new Date(), + }, + ); + } + + async markAsEvaluationRunning( + testRunId: string, + pastExecutionId: string, + evaluationExecutionId: string, + ) { + return await this.update( + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'evaluation_running', + evaluationExecutionId, + }, + ); + } + + async markAsCompleted( + testRunId: string, + pastExecutionId: string, + metrics: Record, + trx?: EntityManager, + ) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'success', + completedAt: new Date(), + metrics, + }, + ); + } + + async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) }, + { + status: 'cancelled', + completedAt: new Date(), + }, + ); + } + + async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'error', + completedAt: new Date(), + }, + ); + } +} diff --git a/packages/cli/src/databases/repositories/test-run.repository.ee.ts b/packages/cli/src/databases/repositories/test-run.repository.ee.ts index 037844734f..1f2039acee 100644 --- a/packages/cli/src/databases/repositories/test-run.repository.ee.ts +++ b/packages/cli/src/databases/repositories/test-run.repository.ee.ts @@ -1,5 +1,5 @@ import { Service } from '@n8n/di'; -import type { FindManyOptions } from '@n8n/typeorm'; +import type { EntityManager, FindManyOptions } from '@n8n/typeorm'; import { DataSource, Repository } from '@n8n/typeorm'; import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee'; @@ -35,12 +35,19 @@ export class TestRunRepository extends Repository { return await this.update(id, { status: 'completed', completedAt: new Date(), metrics }); } - async incrementPassed(id: string) { - return await this.increment({ id }, 'passedCases', 1); + async markAsCancelled(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.update(TestRun, id, { status: 'cancelled' }); } - async incrementFailed(id: string) { - return await this.increment({ id }, 'failedCases', 1); + async incrementPassed(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.increment(TestRun, { id }, 'passedCases', 1); + } + + async incrementFailed(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.increment(TestRun, { id }, 'failedCases', 1); } async getMany(testDefinitionId: string, options: ListQuery.Options) { diff --git a/packages/cli/src/databases/repositories/workflow.repository.ts b/packages/cli/src/databases/repositories/workflow.repository.ts index 7edabdde96..5099a0b526 100644 --- a/packages/cli/src/databases/repositories/workflow.repository.ts +++ b/packages/cli/src/databases/repositories/workflow.repository.ts @@ -12,7 +12,6 @@ import { type FindOptionsRelations, } from '@n8n/typeorm'; -import config from '@/config'; import type { ListQuery } from '@/requests'; import { isStringArray } from '@/utils'; @@ -45,10 +44,12 @@ export class WorkflowRepository extends Repository { }); } - async getActiveIds() { + async getActiveIds({ maxResults }: { maxResults?: number } = {}) { const activeWorkflows = await this.find({ select: ['id'], where: { active: true }, + // 'take' and 'order' are only needed when maxResults is provided: + ...(maxResults ? { take: maxResults, order: { createdAt: 'ASC' } } : {}), }); return activeWorkflows.map((workflow) => workflow.id); } @@ -132,7 +133,7 @@ export class WorkflowRepository extends Repository { const relations: string[] = []; - const areTagsEnabled = !config.getEnv('workflowTagsDisabled'); + const areTagsEnabled = !this.globalConfig.tags.disabled; const isDefaultSelect = options?.select === undefined; const areTagsRequested = isDefaultSelect || options?.select?.tags === true; const isOwnedByIncluded = isDefaultSelect || options?.select?.ownedBy === true; @@ -157,6 +158,11 @@ export class WorkflowRepository extends Repository { findManyOptions.order = { updatedAt: 'ASC' }; } + if (options.sortBy) { + const [column, order] = options.sortBy.split(':'); + findManyOptions.order = { [column]: order }; + } + if (relations.length > 0) { findManyOptions.relations = relations; } diff --git a/packages/cli/src/environments.ee/source-control/__tests__/source-control-export.service.test.ts b/packages/cli/src/environments.ee/source-control/__tests__/source-control-export.service.test.ts index 2e5a5e63e3..96f2f8ecdb 100644 --- a/packages/cli/src/environments.ee/source-control/__tests__/source-control-export.service.test.ts +++ b/packages/cli/src/environments.ee/source-control/__tests__/source-control-export.service.test.ts @@ -1,86 +1,261 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; -import mock from 'jest-mock-extended/lib/Mock'; +import { mock, captor } from 'jest-mock-extended'; import { Cipher, type InstanceSettings } from 'n8n-core'; -import { ApplicationError, deepCopy } from 'n8n-workflow'; +import fsp from 'node:fs/promises'; -import type { CredentialsEntity } from '@/databases/entities/credentials-entity'; import type { SharedCredentials } from '@/databases/entities/shared-credentials'; -import { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository'; -import { mockInstance } from '@test/mocking'; +import type { SharedWorkflow } from '@/databases/entities/shared-workflow'; +import type { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository'; +import type { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository'; +import type { TagRepository } from '@/databases/repositories/tag.repository'; +import type { WorkflowTagMappingRepository } from '@/databases/repositories/workflow-tag-mapping.repository'; +import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import type { VariablesService } from '../../variables/variables.service.ee'; import { SourceControlExportService } from '../source-control-export.service.ee'; -import type { SourceControlledFile } from '../types/source-controlled-file'; - -// https://github.com/jestjs/jest/issues/4715 -function deepSpyOn(object: O, methodName: M) { - const spy = jest.fn(); - - const originalMethod = object[methodName]; - - if (typeof originalMethod !== 'function') { - throw new ApplicationError(`${methodName.toString()} is not a function`, { level: 'warning' }); - } - - object[methodName] = function (...args: unknown[]) { - const clonedArgs = deepCopy(args); - spy(...clonedArgs); - return originalMethod.apply(this, args); - } as O[M]; - - return spy; -} describe('SourceControlExportService', () => { + const cipher = Container.get(Cipher); + const sharedCredentialsRepository = mock(); + const sharedWorkflowRepository = mock(); + const workflowRepository = mock(); + const tagRepository = mock(); + const workflowTagMappingRepository = mock(); + const variablesService = mock(); + const service = new SourceControlExportService( mock(), - mock(), - mock(), - mock({ n8nFolder: '' }), + variablesService, + tagRepository, + sharedCredentialsRepository, + sharedWorkflowRepository, + workflowRepository, + workflowTagMappingRepository, + mock({ n8nFolder: '/mock/n8n' }), ); - describe('exportCredentialsToWorkFolder', () => { - it('should export credentials to work folder', async () => { - /** - * Arrange - */ - // @ts-expect-error Private method - const replaceSpy = deepSpyOn(service, 'replaceCredentialData'); + const fsWriteFile = jest.spyOn(fsp, 'writeFile'); - mockInstance(SharedCredentialsRepository).findByCredentialIds.mockResolvedValue([ + beforeEach(() => jest.clearAllMocks()); + + describe('exportCredentialsToWorkFolder', () => { + const credentialData = { + authUrl: 'test', + accessTokenUrl: 'test', + clientId: 'test', + clientSecret: 'test', + oauthTokenData: { + access_token: 'test', + token_type: 'test', + expires_in: 123, + refresh_token: 'test', + }, + }; + + const mockCredentials = mock({ + id: 'cred1', + name: 'Test Credential', + type: 'oauth2', + data: cipher.encrypt(credentialData), + }); + + it('should export credentials to work folder', async () => { + sharedCredentialsRepository.findByCredentialIds.mockResolvedValue([ mock({ - credentials: mock({ - data: Container.get(Cipher).encrypt( - JSON.stringify({ - authUrl: 'test', - accessTokenUrl: 'test', - clientId: 'test', - clientSecret: 'test', - oauthTokenData: { - access_token: 'test', - token_type: 'test', - expires_in: 123, - refresh_token: 'test', - }, - }), - ), + credentials: mockCredentials, + project: mock({ + type: 'personal', + projectRelations: [ + { + role: 'project:personalOwner', + user: mock({ email: 'user@example.com' }), + }, + ], }), }), ]); - /** - * Act - */ - await service.exportCredentialsToWorkFolder([mock()]); + // Act + const result = await service.exportCredentialsToWorkFolder([mock()]); - /** - * Assert - */ - expect(replaceSpy).toHaveBeenCalledWith({ - authUrl: 'test', - accessTokenUrl: 'test', - clientId: 'test', - clientSecret: 'test', + // Assert + expect(result.count).toBe(1); + expect(result.files).toHaveLength(1); + + const dataCaptor = captor(); + expect(fsWriteFile).toHaveBeenCalledWith( + '/mock/n8n/git/credential_stubs/cred1.json', + dataCaptor, + ); + expect(JSON.parse(dataCaptor.value)).toEqual({ + id: 'cred1', + name: 'Test Credential', + type: 'oauth2', + data: { + authUrl: '', + accessTokenUrl: '', + clientId: '', + clientSecret: '', + }, + ownedBy: { + type: 'personal', + personalEmail: 'user@example.com', + }, }); }); + + it('should handle team project credentials', async () => { + sharedCredentialsRepository.findByCredentialIds.mockResolvedValue([ + mock({ + credentials: mockCredentials, + project: mock({ + type: 'team', + id: 'team1', + name: 'Test Team', + }), + }), + ]); + + // Act + const result = await service.exportCredentialsToWorkFolder([ + mock({ id: 'cred1' }), + ]); + + // Assert + expect(result.count).toBe(1); + + const dataCaptor = captor(); + expect(fsWriteFile).toHaveBeenCalledWith( + '/mock/n8n/git/credential_stubs/cred1.json', + dataCaptor, + ); + expect(JSON.parse(dataCaptor.value)).toEqual({ + id: 'cred1', + name: 'Test Credential', + type: 'oauth2', + data: { + authUrl: '', + accessTokenUrl: '', + clientId: '', + clientSecret: '', + }, + ownedBy: { + type: 'team', + teamId: 'team1', + teamName: 'Test Team', + }, + }); + }); + + it('should handle missing credentials', async () => { + // Arrange + sharedCredentialsRepository.findByCredentialIds.mockResolvedValue([]); + + // Act + const result = await service.exportCredentialsToWorkFolder([ + mock({ id: 'cred1' }), + ]); + + // Assert + expect(result.missingIds).toHaveLength(1); + expect(result.missingIds?.[0]).toBe('cred1'); + }); + }); + + describe('exportTagsToWorkFolder', () => { + it('should export tags to work folder', async () => { + // Arrange + tagRepository.find.mockResolvedValue([mock()]); + workflowTagMappingRepository.find.mockResolvedValue([mock()]); + + // Act + const result = await service.exportTagsToWorkFolder(); + + // Assert + expect(result.count).toBe(1); + expect(result.files).toHaveLength(1); + }); + + it('should not export empty tags', async () => { + // Arrange + tagRepository.find.mockResolvedValue([]); + + // Act + const result = await service.exportTagsToWorkFolder(); + + // Assert + expect(result.count).toBe(0); + expect(result.files).toHaveLength(0); + }); + }); + + describe('exportVariablesToWorkFolder', () => { + it('should export variables to work folder', async () => { + // Arrange + variablesService.getAllCached.mockResolvedValue([mock()]); + + // Act + const result = await service.exportVariablesToWorkFolder(); + + // Assert + expect(result.count).toBe(1); + expect(result.files).toHaveLength(1); + }); + + it('should not export empty variables', async () => { + // Arrange + variablesService.getAllCached.mockResolvedValue([]); + + // Act + const result = await service.exportVariablesToWorkFolder(); + + // Assert + expect(result.count).toBe(0); + expect(result.files).toHaveLength(0); + }); + }); + + describe('exportWorkflowsToWorkFolder', () => { + it('should export workflows to work folder', async () => { + // Arrange + workflowRepository.findByIds.mockResolvedValue([mock()]); + sharedWorkflowRepository.findByWorkflowIds.mockResolvedValue([ + mock({ + project: mock({ + type: 'personal', + projectRelations: [{ role: 'project:personalOwner', user: mock() }], + }), + workflow: mock(), + }), + ]); + + // Act + const result = await service.exportWorkflowsToWorkFolder([mock()]); + + // Assert + expect(result.count).toBe(1); + expect(result.files).toHaveLength(1); + }); + + it('should throw an error if workflow has no owner', async () => { + // Arrange + sharedWorkflowRepository.findByWorkflowIds.mockResolvedValue([ + mock({ + project: mock({ + type: 'personal', + projectRelations: [], + }), + workflow: mock({ + display: () => 'TestWorkflow', + }), + }), + ]); + + // Act & Assert + await expect(service.exportWorkflowsToWorkFolder([mock()])).rejects.toThrow( + 'Workflow TestWorkflow has no owner', + ); + }); }); }); diff --git a/packages/cli/src/environments.ee/source-control/__tests__/source-control-helper.ee.test.ts b/packages/cli/src/environments.ee/source-control/__tests__/source-control-helper.ee.test.ts index c6d1b3857b..4824b85ba6 100644 --- a/packages/cli/src/environments.ee/source-control/__tests__/source-control-helper.ee.test.ts +++ b/packages/cli/src/environments.ee/source-control/__tests__/source-control-helper.ee.test.ts @@ -1,5 +1,7 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { constants as fsConstants, accessSync } from 'fs'; +import { mock } from 'jest-mock-extended'; import { InstanceSettings } from 'n8n-core'; import path from 'path'; @@ -15,11 +17,8 @@ import { getTrackingInformationFromPullResult, sourceControlFoldersExistCheck, } from '@/environments.ee/source-control/source-control-helper.ee'; -import { SourceControlPreferencesService } from '@/environments.ee/source-control/source-control-preferences.service.ee'; -import type { SourceControlPreferences } from '@/environments.ee/source-control/types/source-control-preferences'; -import type { SourceControlledFile } from '@/environments.ee/source-control/types/source-controlled-file'; -import { License } from '@/license'; -import { mockInstance } from '@test/mocking'; +import type { SourceControlPreferencesService } from '@/environments.ee/source-control/source-control-preferences.service.ee'; +import type { License } from '@/license'; const pushResult: SourceControlledFile[] = [ { @@ -151,12 +150,13 @@ const pullResult: SourceControlledFile[] = [ }, ]; -const license = mockInstance(License); +const license = mock(); +const sourceControlPreferencesService = mock(); beforeAll(async () => { jest.resetAllMocks(); license.isSourceControlLicensed.mockReturnValue(true); - Container.get(SourceControlPreferencesService).getPreferences = () => ({ + sourceControlPreferencesService.getPreferences.mockReturnValue({ branchName: 'main', connected: true, repositoryUrl: 'git@example.com:n8ntest/n8n_testrepo.git', @@ -245,17 +245,4 @@ describe('Source Control', () => { workflowUpdates: 3, }); }); - - it('should class validate correct preferences', async () => { - const validPreferences: Partial = { - branchName: 'main', - repositoryUrl: 'git@example.com:n8ntest/n8n_testrepo.git', - branchReadOnly: false, - branchColor: '#5296D6', - }; - const validationResult = await Container.get( - SourceControlPreferencesService, - ).validateSourceControlPreferences(validPreferences); - expect(validationResult).toBeTruthy(); - }); }); diff --git a/packages/cli/src/environments.ee/source-control/__tests__/source-control-import.service.ee.test.ts b/packages/cli/src/environments.ee/source-control/__tests__/source-control-import.service.ee.test.ts new file mode 100644 index 0000000000..cc817368cd --- /dev/null +++ b/packages/cli/src/environments.ee/source-control/__tests__/source-control-import.service.ee.test.ts @@ -0,0 +1,183 @@ +import * as fastGlob from 'fast-glob'; +import { mock } from 'jest-mock-extended'; +import { type InstanceSettings } from 'n8n-core'; +import fsp from 'node:fs/promises'; + +import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; +import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; + +import { SourceControlImportService } from '../source-control-import.service.ee'; + +jest.mock('fast-glob'); + +describe('SourceControlImportService', () => { + const workflowRepository = mock(); + const service = new SourceControlImportService( + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + mock(), + workflowRepository, + mock(), + mock(), + mock(), + mock(), + mock({ n8nFolder: '/mock/n8n' }), + ); + + const globMock = fastGlob.default as unknown as jest.Mock, string[]>; + const fsReadFile = jest.spyOn(fsp, 'readFile'); + + beforeEach(() => jest.clearAllMocks()); + + describe('getRemoteVersionIdsFromFiles', () => { + const mockWorkflowFile = '/mock/workflow1.json'; + it('should parse workflow files correctly', async () => { + globMock.mockResolvedValue([mockWorkflowFile]); + + const mockWorkflowData = { + id: 'workflow1', + versionId: 'v1', + name: 'Test Workflow', + }; + + fsReadFile.mockResolvedValue(JSON.stringify(mockWorkflowData)); + + const result = await service.getRemoteVersionIdsFromFiles(); + expect(fsReadFile).toHaveBeenCalledWith(mockWorkflowFile, { encoding: 'utf8' }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual( + expect.objectContaining({ + id: 'workflow1', + versionId: 'v1', + name: 'Test Workflow', + }), + ); + }); + + it('should filter out files without valid workflow data', async () => { + globMock.mockResolvedValue(['/mock/invalid.json']); + + fsReadFile.mockResolvedValue('{}'); + + const result = await service.getRemoteVersionIdsFromFiles(); + + expect(result).toHaveLength(0); + }); + }); + + describe('getRemoteCredentialsFromFiles', () => { + it('should parse credential files correctly', async () => { + globMock.mockResolvedValue(['/mock/credential1.json']); + + const mockCredentialData = { + id: 'cred1', + name: 'Test Credential', + type: 'oauth2', + }; + + fsReadFile.mockResolvedValue(JSON.stringify(mockCredentialData)); + + const result = await service.getRemoteCredentialsFromFiles(); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual( + expect.objectContaining({ + id: 'cred1', + name: 'Test Credential', + type: 'oauth2', + }), + ); + }); + + it('should filter out files without valid credential data', async () => { + globMock.mockResolvedValue(['/mock/invalid.json']); + fsReadFile.mockResolvedValue('{}'); + + const result = await service.getRemoteCredentialsFromFiles(); + + expect(result).toHaveLength(0); + }); + }); + + describe('getRemoteVariablesFromFile', () => { + it('should parse variables file correctly', async () => { + globMock.mockResolvedValue(['/mock/variables.json']); + + const mockVariablesData = [ + { key: 'VAR1', value: 'value1' }, + { key: 'VAR2', value: 'value2' }, + ]; + + fsReadFile.mockResolvedValue(JSON.stringify(mockVariablesData)); + + const result = await service.getRemoteVariablesFromFile(); + + expect(result).toEqual(mockVariablesData); + }); + + it('should return empty array if no variables file found', async () => { + globMock.mockResolvedValue([]); + + const result = await service.getRemoteVariablesFromFile(); + + expect(result).toHaveLength(0); + }); + }); + + describe('getRemoteTagsAndMappingsFromFile', () => { + it('should parse tags and mappings file correctly', async () => { + globMock.mockResolvedValue(['/mock/tags.json']); + + const mockTagsData = { + tags: [{ id: 'tag1', name: 'Tag 1' }], + mappings: [{ workflowId: 'workflow1', tagId: 'tag1' }], + }; + + fsReadFile.mockResolvedValue(JSON.stringify(mockTagsData)); + + const result = await service.getRemoteTagsAndMappingsFromFile(); + + expect(result.tags).toEqual(mockTagsData.tags); + expect(result.mappings).toEqual(mockTagsData.mappings); + }); + + it('should return empty tags and mappings if no file found', async () => { + globMock.mockResolvedValue([]); + + const result = await service.getRemoteTagsAndMappingsFromFile(); + + expect(result.tags).toHaveLength(0); + expect(result.mappings).toHaveLength(0); + }); + }); + + describe('getLocalVersionIdsFromDb', () => { + const now = new Date(); + jest.useFakeTimers({ now }); + + it('should replace invalid updatedAt with current timestamp', async () => { + const mockWorkflows = [ + { + id: 'workflow1', + name: 'Test Workflow', + updatedAt: 'invalid-date', + }, + ] as unknown as WorkflowEntity[]; + + workflowRepository.find.mockResolvedValue(mockWorkflows); + + const result = await service.getLocalVersionIdsFromDb(); + + expect(result[0].updatedAt).toBe(now.toISOString()); + }); + }); +}); diff --git a/packages/cli/src/environments.ee/source-control/__tests__/source-control-preferences.service.ee.test.ts b/packages/cli/src/environments.ee/source-control/__tests__/source-control-preferences.service.ee.test.ts new file mode 100644 index 0000000000..d35c7ac9c7 --- /dev/null +++ b/packages/cli/src/environments.ee/source-control/__tests__/source-control-preferences.service.ee.test.ts @@ -0,0 +1,27 @@ +import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; + +import { SourceControlPreferencesService } from '../source-control-preferences.service.ee'; +import type { SourceControlPreferences } from '../types/source-control-preferences'; + +describe('SourceControlPreferencesService', () => { + const instanceSettings = mock({ n8nFolder: '' }); + const service = new SourceControlPreferencesService( + instanceSettings, + mock(), + mock(), + mock(), + mock(), + ); + + it('should class validate correct preferences', async () => { + const validPreferences: Partial = { + branchName: 'main', + repositoryUrl: 'git@example.com:n8ntest/n8n_testrepo.git', + branchReadOnly: false, + branchColor: '#5296D6', + }; + const validationResult = await service.validateSourceControlPreferences(validPreferences); + expect(validationResult).toBeTruthy(); + }); +}); diff --git a/packages/cli/src/environments.ee/source-control/__tests__/source-control.service.test.ts b/packages/cli/src/environments.ee/source-control/__tests__/source-control.service.test.ts index 9024a0a32c..43a13ed6ba 100644 --- a/packages/cli/src/environments.ee/source-control/__tests__/source-control.service.test.ts +++ b/packages/cli/src/environments.ee/source-control/__tests__/source-control.service.test.ts @@ -1,30 +1,46 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { mock } from 'jest-mock-extended'; import { InstanceSettings } from 'n8n-core'; +import type { TagEntity } from '@/databases/entities/tag-entity'; +import type { User } from '@/databases/entities/user'; +import type { Variables } from '@/databases/entities/variables'; +import type { TagRepository } from '@/databases/repositories/tag.repository'; import { SourceControlPreferencesService } from '@/environments.ee/source-control/source-control-preferences.service.ee'; import { SourceControlService } from '@/environments.ee/source-control/source-control.service.ee'; +import type { SourceControlImportService } from '../source-control-import.service.ee'; +import type { ExportableCredential } from '../types/exportable-credential'; +import type { SourceControlWorkflowVersionId } from '../types/source-control-workflow-version-id'; + describe('SourceControlService', () => { const preferencesService = new SourceControlPreferencesService( Container.get(InstanceSettings), mock(), mock(), + mock(), + mock(), ); + const sourceControlImportService = mock(); + const tagRepository = mock(); const sourceControlService = new SourceControlService( mock(), mock(), preferencesService, mock(), - mock(), - mock(), + sourceControlImportService, + tagRepository, mock(), ); + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(sourceControlService, 'sanityCheck').mockResolvedValue(undefined); + }); + describe('pushWorkfolder', () => { it('should throw an error if a file is given that is not in the workfolder', async () => { - jest.spyOn(sourceControlService, 'sanityCheck').mockResolvedValue(undefined); - await expect( sourceControlService.pushWorkfolder({ fileNames: [ @@ -44,4 +60,155 @@ describe('SourceControlService', () => { ).rejects.toThrow('File path /etc/passwd is invalid'); }); }); + + describe('pullWorkfolder', () => { + it('does not filter locally created credentials', async () => { + // ARRANGE + const user = mock(); + const statuses = [ + mock({ + status: 'created', + location: 'local', + type: 'credential', + }), + mock({ + status: 'created', + location: 'local', + type: 'workflow', + }), + ]; + jest.spyOn(sourceControlService, 'getStatus').mockResolvedValueOnce(statuses); + + // ACT + const result = await sourceControlService.pullWorkfolder(user, {}); + + // ASSERT + expect(result).toMatchObject({ statusCode: 409, statusResult: statuses }); + }); + + it('does not filter remotely deleted credentials', async () => { + // ARRANGE + const user = mock(); + const statuses = [ + mock({ + status: 'deleted', + location: 'remote', + type: 'credential', + }), + mock({ + status: 'created', + location: 'local', + type: 'workflow', + }), + ]; + jest.spyOn(sourceControlService, 'getStatus').mockResolvedValueOnce(statuses); + + // ACT + const result = await sourceControlService.pullWorkfolder(user, {}); + + // ASSERT + expect(result).toMatchObject({ statusCode: 409, statusResult: statuses }); + }); + + it('should throw an error if a file is given that is not in the workfolder', async () => { + await expect( + sourceControlService.pushWorkfolder({ + fileNames: [ + { + file: '/etc/passwd', + id: 'test', + name: 'secret-file', + type: 'file', + status: 'modified', + location: 'local', + conflict: false, + updatedAt: new Date().toISOString(), + pushed: false, + }, + ], + }), + ).rejects.toThrow('File path /etc/passwd is invalid'); + }); + }); + + describe('getStatus', () => { + it('conflict depends on the value of `direction`', async () => { + // ARRANGE + + // Define a credential that does only exist locally. + // Pulling this would delete it so it should be marked as a conflict. + // Pushing this is conflict free. + sourceControlImportService.getRemoteVersionIdsFromFiles.mockResolvedValue([]); + sourceControlImportService.getLocalVersionIdsFromDb.mockResolvedValue([ + mock(), + ]); + + // Define a credential that does only exist locally. + // Pulling this would delete it so it should be marked as a conflict. + // Pushing this is conflict free. + sourceControlImportService.getRemoteCredentialsFromFiles.mockResolvedValue([]); + sourceControlImportService.getLocalCredentialsFromDb.mockResolvedValue([ + mock(), + ]); + + // Define a variable that does only exist locally. + // Pulling this would delete it so it should be marked as a conflict. + // Pushing this is conflict free. + sourceControlImportService.getRemoteVariablesFromFile.mockResolvedValue([]); + sourceControlImportService.getLocalVariablesFromDb.mockResolvedValue([mock()]); + + // Define a tag that does only exist locally. + // Pulling this would delete it so it should be marked as a conflict. + // Pushing this is conflict free. + const tag = mock({ updatedAt: new Date() }); + tagRepository.find.mockResolvedValue([tag]); + sourceControlImportService.getRemoteTagsAndMappingsFromFile.mockResolvedValue({ + tags: [], + mappings: [], + }); + sourceControlImportService.getLocalTagsAndMappingsFromDb.mockResolvedValue({ + tags: [tag], + mappings: [], + }); + + // ACT + const pullResult = await sourceControlService.getStatus({ + direction: 'pull', + verbose: false, + preferLocalVersion: false, + }); + + const pushResult = await sourceControlService.getStatus({ + direction: 'push', + verbose: false, + preferLocalVersion: false, + }); + + // ASSERT + console.log(pullResult); + console.log(pushResult); + + if (!Array.isArray(pullResult)) { + fail('Expected pullResult to be an array.'); + } + if (!Array.isArray(pushResult)) { + fail('Expected pushResult to be an array.'); + } + + expect(pullResult).toHaveLength(4); + expect(pushResult).toHaveLength(4); + + expect(pullResult.find((i) => i.type === 'workflow')).toHaveProperty('conflict', true); + expect(pushResult.find((i) => i.type === 'workflow')).toHaveProperty('conflict', false); + + expect(pullResult.find((i) => i.type === 'credential')).toHaveProperty('conflict', true); + expect(pushResult.find((i) => i.type === 'credential')).toHaveProperty('conflict', false); + + expect(pullResult.find((i) => i.type === 'variables')).toHaveProperty('conflict', true); + expect(pushResult.find((i) => i.type === 'variables')).toHaveProperty('conflict', false); + + expect(pullResult.find((i) => i.type === 'tags')).toHaveProperty('conflict', true); + expect(pushResult.find((i) => i.type === 'tags')).toHaveProperty('conflict', false); + }); + }); }); diff --git a/packages/cli/src/environments.ee/source-control/source-control-export.service.ee.ts b/packages/cli/src/environments.ee/source-control/source-control-export.service.ee.ts index 4980460318..ec8e4efd22 100644 --- a/packages/cli/src/environments.ee/source-control/source-control-export.service.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control-export.service.ee.ts @@ -1,4 +1,5 @@ -import { Container, Service } from '@n8n/di'; +import type { SourceControlledFile } from '@n8n/api-types'; +import { Service } from '@n8n/di'; import { rmSync } from 'fs'; import { Credentials, InstanceSettings, Logger } from 'n8n-core'; import { ApplicationError, type ICredentialDataDecryptedObject } from 'n8n-workflow'; @@ -29,7 +30,6 @@ import type { ExportResult } from './types/export-result'; import type { ExportableCredential } from './types/exportable-credential'; import type { ExportableWorkflow } from './types/exportable-workflow'; import type { ResourceOwner } from './types/resource-owner'; -import type { SourceControlledFile } from './types/source-controlled-file'; import { VariablesService } from '../variables/variables.service.ee'; @Service() @@ -44,6 +44,10 @@ export class SourceControlExportService { private readonly logger: Logger, private readonly variablesService: VariablesService, private readonly tagRepository: TagRepository, + private readonly sharedCredentialsRepository: SharedCredentialsRepository, + private readonly sharedWorkflowRepository: SharedWorkflowRepository, + private readonly workflowRepository: WorkflowRepository, + private readonly workflowTagMappingRepository: WorkflowTagMappingRepository, instanceSettings: InstanceSettings, ) { this.gitFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_GIT_FOLDER); @@ -106,17 +110,16 @@ export class SourceControlExportService { try { sourceControlFoldersExistCheck([this.workflowExportFolder]); const workflowIds = candidates.map((e) => e.id); - const sharedWorkflows = - await Container.get(SharedWorkflowRepository).findByWorkflowIds(workflowIds); - const workflows = await Container.get(WorkflowRepository).findByIds(workflowIds); + const sharedWorkflows = await this.sharedWorkflowRepository.findByWorkflowIds(workflowIds); + const workflows = await this.workflowRepository.findByIds(workflowIds); // determine owner of each workflow to be exported const owners: Record = {}; - sharedWorkflows.forEach((e) => { - const project = e.project; + sharedWorkflows.forEach((sharedWorkflow) => { + const project = sharedWorkflow.project; if (!project) { - throw new ApplicationError(`Workflow ${e.workflow.display()} has no owner`); + throw new ApplicationError(`Workflow ${sharedWorkflow.workflow.display()} has no owner`); } if (project.type === 'personal') { @@ -124,14 +127,16 @@ export class SourceControlExportService { (pr) => pr.role === 'project:personalOwner', ); if (!ownerRelation) { - throw new ApplicationError(`Workflow ${e.workflow.display()} has no owner`); + throw new ApplicationError( + `Workflow ${sharedWorkflow.workflow.display()} has no owner`, + ); } - owners[e.workflowId] = { + owners[sharedWorkflow.workflowId] = { type: 'personal', personalEmail: ownerRelation.user.email, }; } else if (project.type === 'team') { - owners[e.workflowId] = { + owners[sharedWorkflow.workflowId] = { type: 'team', teamId: project.id, teamName: project.name, @@ -156,6 +161,7 @@ export class SourceControlExportService { })), }; } catch (error) { + if (error instanceof ApplicationError) throw error; throw new ApplicationError('Failed to export workflows to work folder', { cause: error }); } } @@ -204,7 +210,7 @@ export class SourceControlExportService { files: [], }; } - const mappings = await Container.get(WorkflowTagMappingRepository).find(); + const mappings = await this.workflowTagMappingRepository.find(); const fileName = path.join(this.gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE); await fsWriteFile( fileName, @@ -260,9 +266,10 @@ export class SourceControlExportService { try { sourceControlFoldersExistCheck([this.credentialExportFolder]); const credentialIds = candidates.map((e) => e.id); - const credentialsToBeExported = await Container.get( - SharedCredentialsRepository, - ).findByCredentialIds(credentialIds, 'credential:owner'); + const credentialsToBeExported = await this.sharedCredentialsRepository.findByCredentialIds( + credentialIds, + 'credential:owner', + ); let missingIds: string[] = []; if (credentialsToBeExported.length !== credentialIds.length) { const foundCredentialIds = credentialsToBeExported.map((e) => e.credentialsId); diff --git a/packages/cli/src/environments.ee/source-control/source-control-helper.ee.ts b/packages/cli/src/environments.ee/source-control/source-control-helper.ee.ts index 031155a5b5..45ca38ed96 100644 --- a/packages/cli/src/environments.ee/source-control/source-control-helper.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control-helper.ee.ts @@ -1,3 +1,4 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { generateKeyPairSync } from 'crypto'; import { constants as fsConstants, mkdirSync, accessSync } from 'fs'; @@ -16,7 +17,6 @@ import { } from './constants'; import type { KeyPair } from './types/key-pair'; import type { KeyPairType } from './types/key-pair-type'; -import type { SourceControlledFile } from './types/source-controlled-file'; export function stringContainsExpression(testString: string): boolean { return /^=.*\{\{.*\}\}/.test(testString); diff --git a/packages/cli/src/environments.ee/source-control/source-control-import.service.ee.ts b/packages/cli/src/environments.ee/source-control/source-control-import.service.ee.ts index 8da041297b..3c416c0b4c 100644 --- a/packages/cli/src/environments.ee/source-control/source-control-import.service.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control-import.service.ee.ts @@ -1,4 +1,5 @@ -import { Container, Service } from '@n8n/di'; +import type { SourceControlledFile } from '@n8n/api-types'; +import { Service } from '@n8n/di'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In } from '@n8n/typeorm'; import glob from 'fast-glob'; @@ -8,9 +9,11 @@ import { readFile as fsReadFile } from 'node:fs/promises'; import path from 'path'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; +import { CredentialsService } from '@/credentials/credentials.service'; import type { Project } from '@/databases/entities/project'; import { SharedCredentials } from '@/databases/entities/shared-credentials'; import type { TagEntity } from '@/databases/entities/tag-entity'; +import type { User } from '@/databases/entities/user'; import type { Variables } from '@/databases/entities/variables'; import type { WorkflowTagMapping } from '@/databases/entities/workflow-tag-mapping'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; @@ -24,7 +27,9 @@ import { WorkflowTagMappingRepository } from '@/databases/repositories/workflow- import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import type { IWorkflowToImport } from '@/interfaces'; import { isUniqueConstraintError } from '@/response-helper'; +import { TagService } from '@/services/tag.service'; import { assertNever } from '@/utils'; +import { WorkflowService } from '@/workflows/workflow.service'; import { SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER, @@ -37,7 +42,6 @@ import { getCredentialExportPath, getWorkflowExportPath } from './source-control import type { ExportableCredential } from './types/exportable-credential'; import type { ResourceOwner } from './types/resource-owner'; import type { SourceControlWorkflowVersionId } from './types/source-control-workflow-version-id'; -import type { SourceControlledFile } from './types/source-controlled-file'; import { VariablesService } from '../variables/variables.service.ee'; @Service() @@ -53,7 +57,18 @@ export class SourceControlImportService { private readonly errorReporter: ErrorReporter, private readonly variablesService: VariablesService, private readonly activeWorkflowManager: ActiveWorkflowManager, + private readonly credentialsRepository: CredentialsRepository, + private readonly projectRepository: ProjectRepository, private readonly tagRepository: TagRepository, + private readonly sharedWorkflowRepository: SharedWorkflowRepository, + private readonly sharedCredentialsRepository: SharedCredentialsRepository, + private readonly userRepository: UserRepository, + private readonly variablesRepository: VariablesRepository, + private readonly workflowRepository: WorkflowRepository, + private readonly workflowTagMappingRepository: WorkflowTagMappingRepository, + private readonly workflowService: WorkflowService, + private readonly credentialsService: CredentialsService, + private readonly tagService: TagService, instanceSettings: InstanceSettings, ) { this.gitFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_GIT_FOLDER); @@ -91,7 +106,7 @@ export class SourceControlImportService { } async getLocalVersionIdsFromDb(): Promise { - const localWorkflows = await Container.get(WorkflowRepository).find({ + const localWorkflows = await this.workflowRepository.find({ select: ['id', 'name', 'versionId', 'updatedAt'], }); return localWorkflows.map((local) => { @@ -146,7 +161,7 @@ export class SourceControlImportService { } async getLocalCredentialsFromDb(): Promise> { - const localCredentials = await Container.get(CredentialsRepository).find({ + const localCredentials = await this.credentialsRepository.find({ select: ['id', 'name', 'type'], }); return localCredentials.map((local) => ({ @@ -201,24 +216,22 @@ export class SourceControlImportService { const localTags = await this.tagRepository.find({ select: ['id', 'name'], }); - const localMappings = await Container.get(WorkflowTagMappingRepository).find({ + const localMappings = await this.workflowTagMappingRepository.find({ select: ['workflowId', 'tagId'], }); return { tags: localTags, mappings: localMappings }; } async importWorkflowFromWorkFolder(candidates: SourceControlledFile[], userId: string) { - const personalProject = - await Container.get(ProjectRepository).getPersonalProjectForUserOrFail(userId); + const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(userId); const workflowManager = this.activeWorkflowManager; const candidateIds = candidates.map((c) => c.id); - const existingWorkflows = await Container.get(WorkflowRepository).findByIds(candidateIds, { + const existingWorkflows = await this.workflowRepository.findByIds(candidateIds, { fields: ['id', 'name', 'versionId', 'active'], }); - const allSharedWorkflows = await Container.get(SharedWorkflowRepository).findWithFields( - candidateIds, - { select: ['workflowId', 'role', 'projectId'] }, - ); + const allSharedWorkflows = await this.sharedWorkflowRepository.findWithFields(candidateIds, { + select: ['workflowId', 'role', 'projectId'], + }); const importWorkflowsResult = []; // Due to SQLite concurrency issues, we cannot save all workflows at once @@ -235,9 +248,7 @@ export class SourceControlImportService { const existingWorkflow = existingWorkflows.find((e) => e.id === importedWorkflow.id); importedWorkflow.active = existingWorkflow?.active ?? false; this.logger.debug(`Updating workflow id ${importedWorkflow.id ?? 'new'}`); - const upsertResult = await Container.get(WorkflowRepository).upsert({ ...importedWorkflow }, [ - 'id', - ]); + const upsertResult = await this.workflowRepository.upsert({ ...importedWorkflow }, ['id']); if (upsertResult?.identifiers?.length !== 1) { throw new ApplicationError('Failed to upsert workflow', { extra: { workflowId: importedWorkflow.id ?? 'new' }, @@ -253,7 +264,7 @@ export class SourceControlImportService { ? await this.findOrCreateOwnerProject(importedWorkflow.owner) : null; - await Container.get(SharedWorkflowRepository).upsert( + await this.sharedWorkflowRepository.upsert( { workflowId: importedWorkflow.id, projectId: remoteOwnerProject?.id ?? personalProject.id, @@ -276,7 +287,7 @@ export class SourceControlImportService { const error = ensureError(e); this.logger.error(`Failed to activate workflow ${existingWorkflow.id}`, { error }); } finally { - await Container.get(WorkflowRepository).update( + await this.workflowRepository.update( { id: existingWorkflow.id }, { versionId: importedWorkflow.versionId }, ); @@ -295,16 +306,15 @@ export class SourceControlImportService { } async importCredentialsFromWorkFolder(candidates: SourceControlledFile[], userId: string) { - const personalProject = - await Container.get(ProjectRepository).getPersonalProjectForUserOrFail(userId); + const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(userId); const candidateIds = candidates.map((c) => c.id); - const existingCredentials = await Container.get(CredentialsRepository).find({ + const existingCredentials = await this.credentialsRepository.find({ where: { id: In(candidateIds), }, select: ['id', 'name', 'type', 'data'], }); - const existingSharedCredentials = await Container.get(SharedCredentialsRepository).find({ + const existingSharedCredentials = await this.sharedCredentialsRepository.find({ select: ['credentialsId', 'role'], where: { credentialsId: In(candidateIds), @@ -336,7 +346,7 @@ export class SourceControlImportService { } this.logger.debug(`Updating credential id ${newCredentialObject.id as string}`); - await Container.get(CredentialsRepository).upsert(newCredentialObject, ['id']); + await this.credentialsRepository.upsert(newCredentialObject, ['id']); const isOwnedLocally = existingSharedCredentials.some( (c) => c.credentialsId === credential.id && c.role === 'credential:owner', @@ -352,7 +362,7 @@ export class SourceControlImportService { newSharedCredential.projectId = remoteOwnerProject?.id ?? personalProject.id; newSharedCredential.role = 'credential:owner'; - await Container.get(SharedCredentialsRepository).upsert({ ...newSharedCredential }, [ + await this.sharedCredentialsRepository.upsert({ ...newSharedCredential }, [ 'credentialsId', 'projectId', ]); @@ -388,7 +398,7 @@ export class SourceControlImportService { const existingWorkflowIds = new Set( ( - await Container.get(WorkflowRepository).find({ + await this.workflowRepository.find({ select: ['id'], }) ).map((e) => e.id), @@ -417,7 +427,7 @@ export class SourceControlImportService { await Promise.all( mappedTags.mappings.map(async (mapping) => { if (!existingWorkflowIds.has(String(mapping.workflowId))) return; - await Container.get(WorkflowTagMappingRepository).upsert( + await this.workflowTagMappingRepository.upsert( { tagId: String(mapping.tagId), workflowId: String(mapping.workflowId) }, { skipUpdateIfNoValuesChanged: true, @@ -464,12 +474,12 @@ export class SourceControlImportService { overriddenKeys.splice(overriddenKeys.indexOf(variable.key), 1); } try { - await Container.get(VariablesRepository).upsert({ ...variable }, ['id']); + await this.variablesRepository.upsert({ ...variable }, ['id']); } catch (errorUpsert) { if (isUniqueConstraintError(errorUpsert as Error)) { this.logger.debug(`Variable ${variable.key} already exists, updating instead`); try { - await Container.get(VariablesRepository).update({ key: variable.key }, { ...variable }); + await this.variablesRepository.update({ key: variable.key }, { ...variable }); } catch (errorUpdate) { this.logger.debug(`Failed to update variable ${variable.key}, skipping`); this.logger.debug((errorUpdate as Error).message); @@ -484,11 +494,11 @@ export class SourceControlImportService { if (overriddenKeys.length > 0 && valueOverrides) { for (const key of overriddenKeys) { result.imported.push(key); - const newVariable = Container.get(VariablesRepository).create({ + const newVariable = this.variablesRepository.create({ key, value: valueOverrides[key], }); - await Container.get(VariablesRepository).save(newVariable, { transaction: false }); + await this.variablesRepository.save(newVariable, { transaction: false }); } } @@ -497,33 +507,55 @@ export class SourceControlImportService { return result; } + async deleteWorkflowsNotInWorkfolder(user: User, candidates: SourceControlledFile[]) { + for (const candidate of candidates) { + await this.workflowService.delete(user, candidate.id); + } + } + + async deleteCredentialsNotInWorkfolder(user: User, candidates: SourceControlledFile[]) { + for (const candidate of candidates) { + await this.credentialsService.delete(user, candidate.id); + } + } + + async deleteVariablesNotInWorkfolder(candidates: SourceControlledFile[]) { + for (const candidate of candidates) { + await this.variablesService.delete(candidate.id); + } + } + + async deleteTagsNotInWorkfolder(candidates: SourceControlledFile[]) { + for (const candidate of candidates) { + await this.tagService.delete(candidate.id); + } + } + private async findOrCreateOwnerProject(owner: ResourceOwner): Promise { - const projectRepository = Container.get(ProjectRepository); - const userRepository = Container.get(UserRepository); if (typeof owner === 'string' || owner.type === 'personal') { const email = typeof owner === 'string' ? owner : owner.personalEmail; - const user = await userRepository.findOne({ + const user = await this.userRepository.findOne({ where: { email }, }); if (!user) { return null; } - return await projectRepository.getPersonalProjectForUserOrFail(user.id); + return await this.projectRepository.getPersonalProjectForUserOrFail(user.id); } else if (owner.type === 'team') { - let teamProject = await projectRepository.findOne({ + let teamProject = await this.projectRepository.findOne({ where: { id: owner.teamId }, }); if (!teamProject) { try { - teamProject = await projectRepository.save( - projectRepository.create({ + teamProject = await this.projectRepository.save( + this.projectRepository.create({ id: owner.teamId, name: owner.teamName, type: 'team', }), ); } catch (e) { - teamProject = await projectRepository.findOne({ + teamProject = await this.projectRepository.findOne({ where: { id: owner.teamId }, }); if (!teamProject) { diff --git a/packages/cli/src/environments.ee/source-control/source-control-preferences.service.ee.ts b/packages/cli/src/environments.ee/source-control/source-control-preferences.service.ee.ts index e530d9d530..d424a844b2 100644 --- a/packages/cli/src/environments.ee/source-control/source-control-preferences.service.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control-preferences.service.ee.ts @@ -1,4 +1,4 @@ -import { Container, Service } from '@n8n/di'; +import { Service } from '@n8n/di'; import type { ValidationError } from 'class-validator'; import { validate } from 'class-validator'; import { rm as fsRm } from 'fs/promises'; @@ -7,7 +7,6 @@ import { ApplicationError, jsonParse } from 'n8n-workflow'; import { writeFile, chmod, readFile } from 'node:fs/promises'; import path from 'path'; -import config from '@/config'; import { SettingsRepository } from '@/databases/repositories/settings.repository'; import { @@ -17,6 +16,7 @@ import { SOURCE_CONTROL_PREFERENCES_DB_KEY, } from './constants'; import { generateSshKeyPair, isSourceControlLicensed } from './source-control-helper.ee'; +import { SourceControlConfig } from './source-control.config'; import type { KeyPairType } from './types/key-pair-type'; import { SourceControlPreferences } from './types/source-control-preferences'; @@ -34,6 +34,8 @@ export class SourceControlPreferencesService { private readonly instanceSettings: InstanceSettings, private readonly logger: Logger, private readonly cipher: Cipher, + private readonly settingsRepository: SettingsRepository, + private readonly sourceControlConfig: SourceControlConfig, ) { this.sshFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_SSH_FOLDER); this.gitFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_GIT_FOLDER); @@ -64,9 +66,7 @@ export class SourceControlPreferencesService { } private async getKeyPairFromDatabase() { - const dbSetting = await Container.get(SettingsRepository).findByKey( - 'features.sourceControl.sshKeys', - ); + const dbSetting = await this.settingsRepository.findByKey('features.sourceControl.sshKeys'); if (!dbSetting?.value) return null; @@ -120,7 +120,7 @@ export class SourceControlPreferencesService { async deleteKeyPair() { try { await fsRm(this.sshFolder, { recursive: true }); - await Container.get(SettingsRepository).delete({ key: 'features.sourceControl.sshKeys' }); + await this.settingsRepository.delete({ key: 'features.sourceControl.sshKeys' }); } catch (e) { const error = e instanceof Error ? e : new Error(`${e}`); this.logger.error(`Failed to delete SSH key pair: ${error.message}`); @@ -133,14 +133,12 @@ export class SourceControlPreferencesService { async generateAndSaveKeyPair(keyPairType?: KeyPairType): Promise { if (!keyPairType) { keyPairType = - this.getPreferences().keyGeneratorType ?? - (config.get('sourceControl.defaultKeyPairType') as KeyPairType) ?? - 'ed25519'; + this.getPreferences().keyGeneratorType ?? this.sourceControlConfig.defaultKeyPairType; } const keyPair = await generateSshKeyPair(keyPairType); try { - await Container.get(SettingsRepository).save({ + await this.settingsRepository.save({ key: 'features.sourceControl.sshKeys', value: JSON.stringify({ encryptedPrivateKey: this.cipher.encrypt(keyPair.privateKey), @@ -211,7 +209,7 @@ export class SourceControlPreferencesService { if (saveToDb) { const settingsValue = JSON.stringify(this._sourceControlPreferences); try { - await Container.get(SettingsRepository).save( + await this.settingsRepository.save( { key: SOURCE_CONTROL_PREFERENCES_DB_KEY, value: settingsValue, @@ -229,7 +227,7 @@ export class SourceControlPreferencesService { async loadFromDbAndApplySourceControlPreferences(): Promise< SourceControlPreferences | undefined > { - const loadedPreferences = await Container.get(SettingsRepository).findOne({ + const loadedPreferences = await this.settingsRepository.findOne({ where: { key: SOURCE_CONTROL_PREFERENCES_DB_KEY }, }); if (loadedPreferences) { diff --git a/packages/cli/src/environments.ee/source-control/source-control.config.ts b/packages/cli/src/environments.ee/source-control/source-control.config.ts new file mode 100644 index 0000000000..04d12dce4f --- /dev/null +++ b/packages/cli/src/environments.ee/source-control/source-control.config.ts @@ -0,0 +1,8 @@ +import { Config, Env } from '@n8n/config'; + +@Config +export class SourceControlConfig { + /** Default SSH key type to use when generating SSH keys. */ + @Env('N8N_SOURCECONTROL_DEFAULT_SSH_KEY_TYPE') + defaultKeyPairType: 'ed25519' | 'rsa' = 'ed25519'; +} diff --git a/packages/cli/src/environments.ee/source-control/source-control.controller.ee.ts b/packages/cli/src/environments.ee/source-control/source-control.controller.ee.ts index 7b8b2a7266..1f243a1447 100644 --- a/packages/cli/src/environments.ee/source-control/source-control.controller.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control.controller.ee.ts @@ -1,9 +1,12 @@ +import { PullWorkFolderRequestDto, PushWorkFolderRequestDto } from '@n8n/api-types'; +import type { SourceControlledFile } from '@n8n/api-types'; import express from 'express'; import type { PullResult } from 'simple-git'; -import { Get, Post, Patch, RestController, GlobalScope } from '@/decorators'; +import { Get, Post, Patch, RestController, GlobalScope, Body } from '@/decorators'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { EventService } from '@/events/event.service'; +import { AuthenticatedRequest } from '@/requests'; import { SOURCE_CONTROL_DEFAULT_BRANCH } from './constants'; import { @@ -17,7 +20,6 @@ import type { ImportResult } from './types/import-result'; import { SourceControlRequest } from './types/requests'; import { SourceControlGetStatus } from './types/source-control-get-status'; import type { SourceControlPreferences } from './types/source-control-preferences'; -import type { SourceControlledFile } from './types/source-controlled-file'; @RestController('/source-control') export class SourceControlController { @@ -164,19 +166,16 @@ export class SourceControlController { @Post('/push-workfolder', { middlewares: [sourceControlLicensedAndEnabledMiddleware] }) @GlobalScope('sourceControl:push') async pushWorkfolder( - req: SourceControlRequest.PushWorkFolder, + req: AuthenticatedRequest, res: express.Response, + @Body payload: PushWorkFolderRequestDto, ): Promise { - if (this.sourceControlPreferencesService.isBranchReadOnly()) { - throw new BadRequestError('Cannot push onto read-only branch.'); - } - try { await this.sourceControlService.setGitUserDetails( `${req.user.firstName} ${req.user.lastName}`, req.user.email, ); - const result = await this.sourceControlService.pushWorkfolder(req.body); + const result = await this.sourceControlService.pushWorkfolder(payload); res.statusCode = result.statusCode; return result.statusResult; } catch (error) { @@ -187,15 +186,12 @@ export class SourceControlController { @Post('/pull-workfolder', { middlewares: [sourceControlLicensedAndEnabledMiddleware] }) @GlobalScope('sourceControl:pull') async pullWorkfolder( - req: SourceControlRequest.PullWorkFolder, + req: AuthenticatedRequest, res: express.Response, + @Body payload: PullWorkFolderRequestDto, ): Promise { try { - const result = await this.sourceControlService.pullWorkfolder({ - force: req.body.force, - variables: req.body.variables, - userId: req.user.id, - }); + const result = await this.sourceControlService.pullWorkfolder(req.user, payload); res.statusCode = result.statusCode; return result.statusResult; } catch (error) { diff --git a/packages/cli/src/environments.ee/source-control/source-control.service.ee.ts b/packages/cli/src/environments.ee/source-control/source-control.service.ee.ts index 3952b6d6b5..2bd040ee3a 100644 --- a/packages/cli/src/environments.ee/source-control/source-control.service.ee.ts +++ b/packages/cli/src/environments.ee/source-control/source-control.service.ee.ts @@ -1,3 +1,8 @@ +import type { + PullWorkFolderRequestDto, + PushWorkFolderRequestDto, + SourceControlledFile, +} from '@n8n/api-types'; import { Service } from '@n8n/di'; import { writeFileSync } from 'fs'; import { Logger } from 'n8n-core'; @@ -34,10 +39,7 @@ import type { ExportableCredential } from './types/exportable-credential'; import type { ImportResult } from './types/import-result'; import type { SourceControlGetStatus } from './types/source-control-get-status'; import type { SourceControlPreferences } from './types/source-control-preferences'; -import type { SourceControllPullOptions } from './types/source-control-pull-work-folder'; -import type { SourceControlPushWorkFolder } from './types/source-control-push-work-folder'; import type { SourceControlWorkflowVersionId } from './types/source-control-workflow-version-id'; -import type { SourceControlledFile } from './types/source-controlled-file'; @Service() export class SourceControlService { @@ -207,7 +209,7 @@ export class SourceControlService { return; } - async pushWorkfolder(options: SourceControlPushWorkFolder): Promise<{ + async pushWorkfolder(options: PushWorkFolderRequestDto): Promise<{ statusCode: number; pushResult: PushResult | undefined; statusResult: SourceControlledFile[]; @@ -299,7 +301,7 @@ export class SourceControlService { } } - await this.gitService.commit(options.message ?? 'Updated Workfolder'); + await this.gitService.commit(options.commitMessage ?? 'Updated Workfolder'); const pushResult = await this.gitService.push({ branch: this.sourceControlPreferencesService.getBranchName(), @@ -320,8 +322,45 @@ export class SourceControlService { }; } + private getConflicts(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((file) => file.conflict || file.status === 'modified'); + } + + private getWorkflowsToImport(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'workflow' && e.status !== 'deleted'); + } + + private getWorkflowsToDelete(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'workflow' && e.status === 'deleted'); + } + + private getCredentialsToImport(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'credential' && e.status !== 'deleted'); + } + + private getCredentialsToDelete(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'credential' && e.status === 'deleted'); + } + + private getTagsToImport(files: SourceControlledFile[]): SourceControlledFile | undefined { + return files.find((e) => e.type === 'tags' && e.status !== 'deleted'); + } + + private getTagsToDelete(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'tags' && e.status === 'deleted'); + } + + private getVariablesToImport(files: SourceControlledFile[]): SourceControlledFile | undefined { + return files.find((e) => e.type === 'variables' && e.status !== 'deleted'); + } + + private getVariablesToDelete(files: SourceControlledFile[]): SourceControlledFile[] { + return files.filter((e) => e.type === 'variables' && e.status === 'deleted'); + } + async pullWorkfolder( - options: SourceControllPullOptions, + user: User, + options: PullWorkFolderRequestDto, ): Promise<{ statusCode: number; statusResult: SourceControlledFile[] }> { await this.sanityCheck(); @@ -331,58 +370,51 @@ export class SourceControlService { preferLocalVersion: false, })) as SourceControlledFile[]; - // filter out items that will not effect a local change and thus should not - // trigger a conflict warning in the frontend - const filteredResult = statusResult.filter((e) => { - // locally created credentials will not create a conflict on pull - if (e.status === 'created' && e.location === 'local') { - return false; - } - // remotely deleted credentials will not delete local credentials - if (e.type === 'credential' && e.status === 'deleted') { - return false; - } - return true; - }); - if (options.force !== true) { - const possibleConflicts = filteredResult?.filter( - (file) => (file.conflict || file.status === 'modified') && file.type === 'workflow', - ); + const possibleConflicts = this.getConflicts(statusResult); if (possibleConflicts?.length > 0) { await this.gitService.resetBranch(); return { statusCode: 409, - statusResult: filteredResult, + statusResult, }; } } - const workflowsToBeImported = statusResult.filter( - (e) => e.type === 'workflow' && e.status !== 'deleted', - ); + const workflowsToBeImported = this.getWorkflowsToImport(statusResult); await this.sourceControlImportService.importWorkflowFromWorkFolder( workflowsToBeImported, - options.userId, + user.id, ); - - const credentialsToBeImported = statusResult.filter( - (e) => e.type === 'credential' && e.status !== 'deleted', + const workflowsToBeDeleted = this.getWorkflowsToDelete(statusResult); + await this.sourceControlImportService.deleteWorkflowsNotInWorkfolder( + user, + workflowsToBeDeleted, ); + const credentialsToBeImported = this.getCredentialsToImport(statusResult); await this.sourceControlImportService.importCredentialsFromWorkFolder( credentialsToBeImported, - options.userId, + user.id, + ); + const credentialsToBeDeleted = this.getCredentialsToDelete(statusResult); + await this.sourceControlImportService.deleteCredentialsNotInWorkfolder( + user, + credentialsToBeDeleted, ); - const tagsToBeImported = statusResult.find((e) => e.type === 'tags'); + const tagsToBeImported = this.getTagsToImport(statusResult); if (tagsToBeImported) { await this.sourceControlImportService.importTagsFromWorkFolder(tagsToBeImported); } + const tagsToBeDeleted = this.getTagsToDelete(statusResult); + await this.sourceControlImportService.deleteTagsNotInWorkfolder(tagsToBeDeleted); - const variablesToBeImported = statusResult.find((e) => e.type === 'variables'); + const variablesToBeImported = this.getVariablesToImport(statusResult); if (variablesToBeImported) { await this.sourceControlImportService.importVariablesFromWorkFolder(variablesToBeImported); } + const variablesToBeDeleted = this.getVariablesToDelete(statusResult); + await this.sourceControlImportService.deleteVariablesNotInWorkfolder(variablesToBeDeleted); // #region Tracking Information this.eventService.emit( @@ -393,7 +425,7 @@ export class SourceControlService { return { statusCode: 200, - statusResult: filteredResult, + statusResult, }; } @@ -533,7 +565,7 @@ export class SourceControlService { type: 'workflow', status: options.direction === 'push' ? 'created' : 'deleted', location: options.direction === 'push' ? 'local' : 'remote', - conflict: false, + conflict: options.direction === 'push' ? false : true, file: item.filename, updatedAt: item.updatedAt ?? new Date().toISOString(), }); @@ -614,7 +646,7 @@ export class SourceControlService { type: 'credential', status: options.direction === 'push' ? 'created' : 'deleted', location: options.direction === 'push' ? 'local' : 'remote', - conflict: false, + conflict: options.direction === 'push' ? false : true, file: item.filename, updatedAt: new Date().toISOString(), }); @@ -666,26 +698,47 @@ export class SourceControlService { } }); - if ( - varMissingInLocal.length > 0 || - varMissingInRemote.length > 0 || - varModifiedInEither.length > 0 - ) { - if (options.direction === 'pull' && varRemoteIds.length === 0) { - // if there's nothing to pull, don't show difference as modified - } else { - sourceControlledFiles.push({ - id: 'variables', - name: 'variables', - type: 'variables', - status: 'modified', - location: options.direction === 'push' ? 'local' : 'remote', - conflict: false, - file: getVariablesPath(this.gitFolder), - updatedAt: new Date().toISOString(), - }); - } - } + varMissingInLocal.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.key, + type: 'variables', + status: options.direction === 'push' ? 'deleted' : 'created', + location: options.direction === 'push' ? 'local' : 'remote', + conflict: false, + file: getVariablesPath(this.gitFolder), + updatedAt: new Date().toISOString(), + }); + }); + + varMissingInRemote.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.key, + type: 'variables', + status: options.direction === 'push' ? 'created' : 'deleted', + location: options.direction === 'push' ? 'local' : 'remote', + // if the we pull and the file is missing in the remote, we will delete + // it locally, which is communicated by marking this as a conflict + conflict: options.direction === 'push' ? false : true, + file: getVariablesPath(this.gitFolder), + updatedAt: new Date().toISOString(), + }); + }); + + varModifiedInEither.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.key, + type: 'variables', + status: 'modified', + location: options.direction === 'push' ? 'local' : 'remote', + conflict: true, + file: getVariablesPath(this.gitFolder), + updatedAt: new Date().toISOString(), + }); + }); + return { varMissingInLocal, varMissingInRemote, @@ -740,32 +793,44 @@ export class SourceControlService { ) === -1, ); - if ( - tagsMissingInLocal.length > 0 || - tagsMissingInRemote.length > 0 || - tagsModifiedInEither.length > 0 || - mappingsMissingInLocal.length > 0 || - mappingsMissingInRemote.length > 0 - ) { - if ( - options.direction === 'pull' && - tagMappingsRemote.tags.length === 0 && - tagMappingsRemote.mappings.length === 0 - ) { - // if there's nothing to pull, don't show difference as modified - } else { - sourceControlledFiles.push({ - id: 'mappings', - name: 'tags', - type: 'tags', - status: 'modified', - location: options.direction === 'push' ? 'local' : 'remote', - conflict: false, - file: getTagsPath(this.gitFolder), - updatedAt: lastUpdatedTag[0]?.updatedAt.toISOString(), - }); - } - } + tagsMissingInLocal.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.name, + type: 'tags', + status: options.direction === 'push' ? 'deleted' : 'created', + location: options.direction === 'push' ? 'local' : 'remote', + conflict: false, + file: getTagsPath(this.gitFolder), + updatedAt: lastUpdatedTag[0]?.updatedAt.toISOString(), + }); + }); + tagsMissingInRemote.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.name, + type: 'tags', + status: options.direction === 'push' ? 'created' : 'deleted', + location: options.direction === 'push' ? 'local' : 'remote', + conflict: options.direction === 'push' ? false : true, + file: getTagsPath(this.gitFolder), + updatedAt: lastUpdatedTag[0]?.updatedAt.toISOString(), + }); + }); + + tagsModifiedInEither.forEach((item) => { + sourceControlledFiles.push({ + id: item.id, + name: item.name, + type: 'tags', + status: 'modified', + location: options.direction === 'push' ? 'local' : 'remote', + conflict: true, + file: getTagsPath(this.gitFolder), + updatedAt: lastUpdatedTag[0]?.updatedAt.toISOString(), + }); + }); + return { tagsMissingInLocal, tagsMissingInRemote, diff --git a/packages/cli/src/environments.ee/source-control/types/requests.ts b/packages/cli/src/environments.ee/source-control/types/requests.ts index 4f5d583f40..75c6c1995d 100644 --- a/packages/cli/src/environments.ee/source-control/types/requests.ts +++ b/packages/cli/src/environments.ee/source-control/types/requests.ts @@ -5,9 +5,7 @@ import type { SourceControlDisconnect } from './source-control-disconnect'; import type { SourceControlGenerateKeyPair } from './source-control-generate-key-pair'; import type { SourceControlGetStatus } from './source-control-get-status'; import type { SourceControlPreferences } from './source-control-preferences'; -import type { SourceControlPullWorkFolder } from './source-control-pull-work-folder'; import type { SourceControlPush } from './source-control-push'; -import type { SourceControlPushWorkFolder } from './source-control-push-work-folder'; import type { SourceControlSetBranch } from './source-control-set-branch'; import type { SourceControlSetReadOnly } from './source-control-set-read-only'; import type { SourceControlStage } from './source-control-stage'; @@ -20,8 +18,6 @@ export declare namespace SourceControlRequest { type Stage = AuthenticatedRequest<{}, {}, SourceControlStage, {}>; type Push = AuthenticatedRequest<{}, {}, SourceControlPush, {}>; type Disconnect = AuthenticatedRequest<{}, {}, SourceControlDisconnect, {}>; - type PushWorkFolder = AuthenticatedRequest<{}, {}, SourceControlPushWorkFolder, {}>; - type PullWorkFolder = AuthenticatedRequest<{}, {}, SourceControlPullWorkFolder, {}>; type GetStatus = AuthenticatedRequest<{}, {}, {}, SourceControlGetStatus>; type GenerateKeyPair = AuthenticatedRequest<{}, {}, SourceControlGenerateKeyPair, {}>; } diff --git a/packages/cli/src/environments.ee/source-control/types/source-control-pull-work-folder.ts b/packages/cli/src/environments.ee/source-control/types/source-control-pull-work-folder.ts deleted file mode 100644 index b87c970f0e..0000000000 --- a/packages/cli/src/environments.ee/source-control/types/source-control-pull-work-folder.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { IsBoolean, IsObject, IsOptional, IsString } from 'class-validator'; - -export class SourceControlPullWorkFolder { - @IsBoolean() - @IsOptional() - force?: boolean; - - @IsBoolean() - @IsOptional() - importAfterPull?: boolean = true; - - @IsString({ each: true }) - @IsOptional() - files?: Set; - - @IsObject() - @IsOptional() - variables?: { [key: string]: string }; -} - -export class SourceControllPullOptions { - /** ID of user performing a source control pull. */ - userId: string; - - force?: boolean; - - variables?: { [key: string]: string }; -} diff --git a/packages/cli/src/environments.ee/source-control/types/source-control-push-work-folder.ts b/packages/cli/src/environments.ee/source-control/types/source-control-push-work-folder.ts deleted file mode 100644 index 55c8e178c4..0000000000 --- a/packages/cli/src/environments.ee/source-control/types/source-control-push-work-folder.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { IsBoolean, IsOptional, IsString } from 'class-validator'; - -import type { SourceControlledFile } from './source-controlled-file'; - -export class SourceControlPushWorkFolder { - @IsBoolean() - @IsOptional() - force?: boolean; - - @IsString({ each: true }) - fileNames: SourceControlledFile[]; - - @IsString() - @IsOptional() - message?: string; - - @IsBoolean() - @IsOptional() - skipDiff?: boolean; -} diff --git a/packages/cli/src/environments.ee/source-control/types/source-controlled-file.ts b/packages/cli/src/environments.ee/source-control/types/source-controlled-file.ts deleted file mode 100644 index 5bbf75921b..0000000000 --- a/packages/cli/src/environments.ee/source-control/types/source-controlled-file.ts +++ /dev/null @@ -1,23 +0,0 @@ -export type SourceControlledFileStatus = - | 'new' - | 'modified' - | 'deleted' - | 'created' - | 'renamed' - | 'conflicted' - | 'ignored' - | 'staged' - | 'unknown'; -export type SourceControlledFileLocation = 'local' | 'remote'; -export type SourceControlledFileType = 'credential' | 'workflow' | 'tags' | 'variables' | 'file'; -export type SourceControlledFile = { - file: string; - id: string; - name: string; - type: SourceControlledFileType; - status: SourceControlledFileStatus; - location: SourceControlledFileLocation; - conflict: boolean; - updatedAt: string; - pushed?: boolean; -}; diff --git a/packages/cli/src/environments.ee/variables/environment-helpers.ts b/packages/cli/src/environments.ee/variables/environment-helpers.ts deleted file mode 100644 index dd9a17c95b..0000000000 --- a/packages/cli/src/environments.ee/variables/environment-helpers.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Container } from '@n8n/di'; - -import { License } from '@/license'; - -export function isVariablesEnabled(): boolean { - const license = Container.get(License); - return license.isVariablesEnabled(); -} - -export function canCreateNewVariable(variableCount: number): boolean { - if (!isVariablesEnabled()) { - return false; - } - const license = Container.get(License); - // This defaults to -1 which is what we want if we've enabled - // variables via the config - const limit = license.getVariablesLimit(); - if (limit === -1) { - return true; - } - return limit > variableCount; -} - -export function getVariablesLimit(): number { - const license = Container.get(License); - return license.getVariablesLimit(); -} diff --git a/packages/cli/src/environments.ee/variables/variables.service.ee.ts b/packages/cli/src/environments.ee/variables/variables.service.ee.ts index ebb134efd3..f59880fe60 100644 --- a/packages/cli/src/environments.ee/variables/variables.service.ee.ts +++ b/packages/cli/src/environments.ee/variables/variables.service.ee.ts @@ -1,4 +1,4 @@ -import { Container, Service } from '@n8n/di'; +import { Service } from '@n8n/di'; import type { Variables } from '@/databases/entities/variables'; import { VariablesRepository } from '@/databases/repositories/variables.repository'; @@ -6,23 +6,21 @@ import { generateNanoId } from '@/databases/utils/generators'; import { VariableCountLimitReachedError } from '@/errors/variable-count-limit-reached.error'; import { VariableValidationError } from '@/errors/variable-validation.error'; import { EventService } from '@/events/event.service'; +import { License } from '@/license'; import { CacheService } from '@/services/cache/cache.service'; -import { canCreateNewVariable } from './environment-helpers'; - @Service() export class VariablesService { constructor( - protected cacheService: CacheService, - protected variablesRepository: VariablesRepository, + private readonly cacheService: CacheService, + private readonly variablesRepository: VariablesRepository, private readonly eventService: EventService, + private readonly license: License, ) {} async getAllCached(state?: 'empty'): Promise { let variables = await this.cacheService.get('variables', { - async refreshFn() { - return await Container.get(VariablesService).findAll(); - }, + refreshFn: async () => await this.findAll(), }); if (variables === undefined) { @@ -77,7 +75,7 @@ export class VariablesService { } async create(variable: Omit): Promise { - if (!canCreateNewVariable(await this.getCount())) { + if (!this.canCreateNewVariable(await this.getCount())) { throw new VariableCountLimitReachedError('Variables limit reached'); } this.validateVariable(variable); @@ -100,4 +98,17 @@ export class VariablesService { await this.updateCache(); return (await this.getCached(id))!; } + + private canCreateNewVariable(variableCount: number): boolean { + if (!this.license.isVariablesEnabled()) { + return false; + } + // This defaults to -1 which is what we want if we've enabled + // variables via the config + const limit = this.license.getVariablesLimit(); + if (limit === -1) { + return true; + } + return limit > variableCount; + } } diff --git a/packages/cli/src/errors/max-stalled-count.error.ts b/packages/cli/src/errors/max-stalled-count.error.ts index 653ca18eac..38f73023a7 100644 --- a/packages/cli/src/errors/max-stalled-count.error.ts +++ b/packages/cli/src/errors/max-stalled-count.error.ts @@ -5,9 +5,12 @@ import { ApplicationError } from 'n8n-workflow'; */ export class MaxStalledCountError extends ApplicationError { constructor(cause: Error) { - super('The execution has reached the maximum number of attempts and will no longer retry.', { - level: 'warning', - cause, - }); + super( + 'This execution failed to be processed too many times and will no longer retry. To allow this execution to complete, please break down your workflow or scale up your workers or adjust your worker settings.', + { + level: 'warning', + cause, + }, + ); } } diff --git a/packages/cli/src/errors/response-errors/not-implemented.error.ts b/packages/cli/src/errors/response-errors/not-implemented.error.ts new file mode 100644 index 0000000000..f6c66391ff --- /dev/null +++ b/packages/cli/src/errors/response-errors/not-implemented.error.ts @@ -0,0 +1,7 @@ +import { ResponseError } from './abstract/response.error'; + +export class NotImplementedError extends ResponseError { + constructor(message: string, hint: string | undefined = undefined) { + super(message, 501, 501, hint); + } +} diff --git a/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts b/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts index b7441a2763..36d9715acc 100644 --- a/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts +++ b/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts @@ -92,4 +92,8 @@ export declare namespace TestRunsRequest { type GetOne = AuthenticatedRequest; type Delete = AuthenticatedRequest; + + type Cancel = AuthenticatedRequest; + + type GetCases = AuthenticatedRequest; } diff --git a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts index cc3c3bc33b..7cfe7b3705 100644 --- a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts +++ b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts @@ -13,17 +13,23 @@ import type { TestMetric } from '@/databases/entities/test-metric.ee'; import type { TestRun } from '@/databases/entities/test-run.ee'; import type { User } from '@/databases/entities/user'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import type { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee'; import type { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { NodeTypes } from '@/node-types'; +import type { Telemetry } from '@/telemetry'; import type { WorkflowRunner } from '@/workflow-runner'; -import { mockInstance } from '@test/mocking'; +import { mockInstance, mockLogger } from '@test/mocking'; import { mockNodeTypesData } from '@test-integration/utils/node-types-data'; import { TestRunnerService } from '../test-runner.service.ee'; +jest.mock('@/db', () => ({ + transaction: (cb: any) => cb(), +})); + const wfUnderTestJson = JSON.parse( readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }), ); @@ -129,6 +135,16 @@ function mockEvaluationExecutionData(metrics: Record) { }); } +const errorReporter = mock(); +const logger = mockLogger(); +const telemetry = mock(); + +async function mockLongExecutionPromise(data: IRun, delay: number): Promise { + return await new Promise((resolve) => { + setTimeout(() => resolve(data), delay); + }); +} + describe('TestRunnerService', () => { const executionRepository = mock(); const workflowRepository = mock(); @@ -136,6 +152,7 @@ describe('TestRunnerService', () => { const activeExecutions = mock(); const testRunRepository = mock(); const testMetricRepository = mock(); + const testCaseExecutionRepository = mock(); const mockNodeTypes = mockInstance(NodeTypes); mockInstance(LoadNodesAndCredentials, { @@ -165,25 +182,24 @@ describe('TestRunnerService', () => { }); afterEach(() => { - activeExecutions.getPostExecutePromise.mockClear(); - workflowRunner.run.mockClear(); - testRunRepository.createTestRun.mockClear(); - testRunRepository.markAsRunning.mockClear(); - testRunRepository.markAsCompleted.mockClear(); + jest.resetAllMocks(); testRunRepository.incrementFailed.mockClear(); testRunRepository.incrementPassed.mockClear(); }); test('should create an instance of TestRunnerService', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); expect(testRunnerService).toBeInstanceOf(TestRunnerService); @@ -191,14 +207,17 @@ describe('TestRunnerService', () => { test('should create and run test cases from past executions', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -229,14 +248,17 @@ describe('TestRunnerService', () => { test('should run both workflow under test and evaluation workflow', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -330,14 +352,17 @@ describe('TestRunnerService', () => { test('should properly count passed and failed executions', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -388,14 +413,17 @@ describe('TestRunnerService', () => { test('should properly count failed test executions', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -442,14 +470,17 @@ describe('TestRunnerService', () => { test('should properly count failed evaluations', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -500,14 +531,17 @@ describe('TestRunnerService', () => { test('should specify correct start nodes when running workflow under test', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -574,14 +608,17 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); const startNodesData = (testRunnerService as any).getStartNodesData( @@ -599,14 +636,17 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes 2', async () => { const testRunnerService = new TestRunnerService( + logger, + telemetry, workflowRepository, workflowRunner, executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); const startNodesData = (testRunnerService as any).getStartNodesData( @@ -621,4 +661,89 @@ describe('TestRunnerService', () => { }), }); }); + + describe('Test Run cancellation', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + test('should cancel test run', async () => { + const testRunnerService = new TestRunnerService( + logger, + telemetry, + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + testRunRepository, + testCaseExecutionRepository, + testMetricRepository, + mockNodeTypes, + errorReporter, + ); + + workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ + id: 'workflow-under-test-id', + ...wfUnderTestJson, + }); + + workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({ + id: 'evaluation-workflow-id', + ...wfEvaluationJson, + }); + + workflowRunner.run.mockResolvedValueOnce('some-execution-id'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-2'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-3'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-4'); + + // Mock long execution of workflow under test + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id') + .mockReturnValue(mockLongExecutionPromise(mockExecutionData(), 1000)); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-3') + .mockReturnValue(mockLongExecutionPromise(mockExecutionData(), 1000)); + + // Mock executions of evaluation workflow + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-2') + .mockReturnValue( + mockLongExecutionPromise(mockEvaluationExecutionData({ metric1: 1, metric2: 0 }), 1000), + ); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-4') + .mockReturnValue( + mockLongExecutionPromise(mockEvaluationExecutionData({ metric1: 0.5 }), 1000), + ); + + // Do not await here to test canceling + void testRunnerService.runTest( + mock(), + mock({ + workflowId: 'workflow-under-test-id', + evaluationWorkflowId: 'evaluation-workflow-id', + mockedNodes: [{ id: '72256d90-3a67-4e29-b032-47df4e5768af' }], + }), + ); + + // Simulate the moment when first test case is running (wf under test execution) + await jest.advanceTimersByTimeAsync(100); + expect(workflowRunner.run).toHaveBeenCalledTimes(1); + + const abortController = (testRunnerService as any).abortControllers.get('test-run-id'); + expect(abortController).toBeDefined(); + + await testRunnerService.cancelTestRun('test-run-id'); + + expect(abortController.signal.aborted).toBe(true); + expect(activeExecutions.stopExecution).toBeCalledWith('some-execution-id'); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + }); }); diff --git a/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts b/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts index ab5c921f8c..cd320bd8da 100644 --- a/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts @@ -9,12 +9,17 @@ export class EvaluationMetrics { } } - addResults(result: IDataObject) { + addResults(result: IDataObject): Record { + const addedMetrics: Record = {}; + for (const [metricName, metricValue] of Object.entries(result)) { if (typeof metricValue === 'number' && this.metricNames.has(metricName)) { + addedMetrics[metricName] = metricValue; this.rawMetricsByName.get(metricName)!.push(metricValue); } } + + return addedMetrics; } getAggregatedMetrics() { diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index 5a054a3527..09c4fe4ec6 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -1,7 +1,7 @@ import { Service } from '@n8n/di'; import { parse } from 'flatted'; -import { ErrorReporter } from 'n8n-core'; -import { NodeConnectionType, Workflow } from 'n8n-workflow'; +import { ErrorReporter, Logger } from 'n8n-core'; +import { ExecutionCancelledError, NodeConnectionType, Workflow } from 'n8n-workflow'; import type { IDataObject, IRun, @@ -15,19 +15,32 @@ import assert from 'node:assert'; import { ActiveExecutions } from '@/active-executions'; import type { ExecutionEntity } from '@/databases/entities/execution-entity'; import type { MockedNodeItem, TestDefinition } from '@/databases/entities/test-definition.ee'; +import type { TestRun } from '@/databases/entities/test-run.ee'; import type { User } from '@/databases/entities/user'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee'; import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import * as Db from '@/db'; import { NodeTypes } from '@/node-types'; +import { Telemetry } from '@/telemetry'; import { getRunData } from '@/workflow-execute-additional-data'; import { WorkflowRunner } from '@/workflow-runner'; import { EvaluationMetrics } from './evaluation-metrics.ee'; import { createPinData, getPastExecutionTriggerNode } from './utils.ee'; +interface TestRunMetadata { + testRunId: string; + userId: string; +} + +interface TestCaseRunMetadata extends TestRunMetadata { + pastExecutionId: string; +} + /** * This service orchestrates the running of test cases. * It uses the test definitions to find @@ -38,12 +51,17 @@ import { createPinData, getPastExecutionTriggerNode } from './utils.ee'; */ @Service() export class TestRunnerService { + private abortControllers: Map = new Map(); + constructor( + private readonly logger: Logger, + private readonly telemetry: Telemetry, private readonly workflowRepository: WorkflowRepository, private readonly workflowRunner: WorkflowRunner, private readonly executionRepository: ExecutionRepository, private readonly activeExecutions: ActiveExecutions, private readonly testRunRepository: TestRunRepository, + private readonly testCaseExecutionRepository: TestCaseExecutionRepository, private readonly testMetricRepository: TestMetricRepository, private readonly nodeTypes: NodeTypes, private readonly errorReporter: ErrorReporter, @@ -99,8 +117,14 @@ export class TestRunnerService { pastExecutionData: IRunExecutionData, pastExecutionWorkflowData: IWorkflowBase, mockedNodes: MockedNodeItem[], - userId: string, + metadata: TestCaseRunMetadata, + abortSignal: AbortSignal, ): Promise { + // Do not run if the test run is cancelled + if (abortSignal.aborted) { + return; + } + // Create pin data from the past execution data const pinData = createPinData( workflow, @@ -115,14 +139,27 @@ export class TestRunnerService { executionMode: 'evaluation', runData: {}, pinData, - workflowData: workflow, - userId, + workflowData: { ...workflow, pinData }, + userId: metadata.userId, + partialExecutionVersion: 2, }; // Trigger the workflow under test with mocked data const executionId = await this.workflowRunner.run(data); assert(executionId); + // Listen to the abort signal to stop the execution in case test run is cancelled + abortSignal.addEventListener('abort', () => { + this.activeExecutions.stopExecution(executionId); + }); + + // Update status of the test run execution mapping + await this.testCaseExecutionRepository.markAsRunning( + metadata.testRunId, + metadata.pastExecutionId, + executionId, + ); + // Wait for the execution to finish const executePromise = this.activeExecutions.getPostExecutePromise(executionId); @@ -136,8 +173,14 @@ export class TestRunnerService { evaluationWorkflow: WorkflowEntity, expectedData: IRunData, actualData: IRunData, - testRunId?: string, + abortSignal: AbortSignal, + metadata: TestCaseRunMetadata, ) { + // Do not run if the test run is cancelled + if (abortSignal.aborted) { + return; + } + // Prepare the evaluation wf input data. // Provide both the expected data and the actual data const evaluationInputData = { @@ -149,19 +192,24 @@ export class TestRunnerService { // Prepare the data to run the evaluation workflow const data = await getRunData(evaluationWorkflow, [evaluationInputData]); - // FIXME: This is a hack to add the testRunId to the evaluation workflow execution data - // So that we can fetch all execution runs for a test run - if (testRunId && data.executionData) { - data.executionData.resultData.metadata = { - testRunId, - }; - } data.executionMode = 'evaluation'; // Trigger the evaluation workflow const executionId = await this.workflowRunner.run(data); assert(executionId); + // Listen to the abort signal to stop the execution in case test run is cancelled + abortSignal.addEventListener('abort', () => { + this.activeExecutions.stopExecution(executionId); + }); + + // Update status of the test run execution mapping + await this.testCaseExecutionRepository.markAsEvaluationRunning( + metadata.testRunId, + metadata.pastExecutionId, + executionId, + ); + // Wait for the execution to finish const executePromise = this.activeExecutions.getPostExecutePromise(executionId); @@ -203,6 +251,8 @@ export class TestRunnerService { * Creates a new test run for the given test definition. */ async runTest(user: User, test: TestDefinition): Promise { + this.logger.debug('Starting new test run', { testId: test.id }); + const workflow = await this.workflowRepository.findById(test.workflowId); assert(workflow, 'Workflow not found'); @@ -213,89 +263,224 @@ export class TestRunnerService { const testRun = await this.testRunRepository.createTestRun(test.id); assert(testRun, 'Unable to create a test run'); - // 1. Make test cases from previous executions + // 0.1 Initialize AbortController + const abortController = new AbortController(); + this.abortControllers.set(testRun.id, abortController); - // Select executions with the annotation tag and workflow ID of the test. - // Fetch only ids to reduce the data transfer. - const pastExecutions: ReadonlyArray> = - await this.executionRepository - .createQueryBuilder('execution') - .select('execution.id') - .leftJoin('execution.annotation', 'annotation') - .leftJoin('annotation.tags', 'annotationTag') - .where('annotationTag.id = :tagId', { tagId: test.annotationTagId }) - .andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId }) - .getMany(); + // 0.2 Initialize metadata + // This will be passed to the test case executions + const testRunMetadata = { + testRunId: testRun.id, + userId: user.id, + }; - // Get the metrics to collect from the evaluation workflow - const testMetricNames = await this.getTestMetricNames(test.id); + const abortSignal = abortController.signal; + try { + /// + // 1. Make test cases from previous executions + /// - // 2. Run over all the test cases + // Select executions with the annotation tag and workflow ID of the test. + // Fetch only ids to reduce the data transfer. + const pastExecutions: ReadonlyArray> = + await this.executionRepository + .createQueryBuilder('execution') + .select('execution.id') + .leftJoin('execution.annotation', 'annotation') + .leftJoin('annotation.tags', 'annotationTag') + .where('annotationTag.id = :tagId', { tagId: test.annotationTagId }) + .andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId }) + .getMany(); - await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); + this.logger.debug('Found past executions', { count: pastExecutions.length }); - // Object to collect the results of the evaluation workflow executions - const metrics = new EvaluationMetrics(testMetricNames); + // Add all past executions mappings to the test run. + // This will be used to track the status of each test case and keep the connection between test run and all related executions (past, current, and evaluation). + await this.testCaseExecutionRepository.createBatch( + testRun.id, + pastExecutions.map((e) => e.id), + ); - for (const { id: pastExecutionId } of pastExecutions) { - try { - // Fetch past execution with data - const pastExecution = await this.executionRepository.findOne({ - where: { id: pastExecutionId }, - relations: ['executionData', 'metadata'], - }); - assert(pastExecution, 'Execution not found'); + // Get the metrics to collect from the evaluation workflow + const testMetricNames = await this.getTestMetricNames(test.id); - const executionData = parse(pastExecution.executionData.data) as IRunExecutionData; + // 2. Run over all the test cases + const pastExecutionIds = pastExecutions.map((e) => e.id); - // Run the test case and wait for it to finish - const testCaseExecution = await this.runTestCase( - workflow, - executionData, - pastExecution.executionData.workflowData, - test.mockedNodes, - user.id, - ); + // Update test run status + await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); - // In case of a permission check issue, the test case execution will be undefined. - // Skip them, increment the failed count and continue with the next test case - if (!testCaseExecution) { - await this.testRunRepository.incrementFailed(testRun.id); - continue; + this.telemetry.track('User runs test', { + user_id: user.id, + test_id: test.id, + run_id: testRun.id, + executions_ids: pastExecutionIds, + workflow_id: test.workflowId, + evaluation_workflow_id: test.evaluationWorkflowId, + }); + + // Initialize object to collect the results of the evaluation workflow executions + const metrics = new EvaluationMetrics(testMetricNames); + + /// + // 2. Run over all the test cases + /// + + for (const pastExecutionId of pastExecutionIds) { + if (abortSignal.aborted) { + this.logger.debug('Test run was cancelled', { + testId: test.id, + stoppedOn: pastExecutionId, + }); + break; } - // Collect the results of the test case execution - const testCaseRunData = testCaseExecution.data.resultData.runData; + this.logger.debug('Running test case', { pastExecutionId }); - // Get the original runData from the test case execution data - const originalRunData = executionData.resultData.runData; + try { + // Fetch past execution with data + const pastExecution = await this.executionRepository.findOne({ + where: { id: pastExecutionId }, + relations: ['executionData', 'metadata'], + }); + assert(pastExecution, 'Execution not found'); - // Run the evaluation workflow with the original and new run data - const evalExecution = await this.runTestCaseEvaluation( - evaluationWorkflow, - originalRunData, - testCaseRunData, - testRun.id, - ); - assert(evalExecution); + const executionData = parse(pastExecution.executionData.data) as IRunExecutionData; - metrics.addResults(this.extractEvaluationResult(evalExecution)); + const testCaseMetadata = { + ...testRunMetadata, + pastExecutionId, + }; - if (evalExecution.data.resultData.error) { - await this.testRunRepository.incrementFailed(testRun.id); - } else { - await this.testRunRepository.incrementPassed(testRun.id); + // Run the test case and wait for it to finish + const testCaseExecution = await this.runTestCase( + workflow, + executionData, + pastExecution.executionData.workflowData, + test.mockedNodes, + testCaseMetadata, + abortSignal, + ); + + this.logger.debug('Test case execution finished', { pastExecutionId }); + + // In case of a permission check issue, the test case execution will be undefined. + // Skip them, increment the failed count and continue with the next test case + if (!testCaseExecution) { + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); + continue; + } + + // Update status of the test case execution mapping entry in case of an error + if (testCaseExecution.data.resultData.error) { + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId); + } + + // Collect the results of the test case execution + const testCaseRunData = testCaseExecution.data.resultData.runData; + + // Get the original runData from the test case execution data + const originalRunData = executionData.resultData.runData; + + // Run the evaluation workflow with the original and new run data + const evalExecution = await this.runTestCaseEvaluation( + evaluationWorkflow, + originalRunData, + testCaseRunData, + abortSignal, + testCaseMetadata, + ); + assert(evalExecution); + + this.logger.debug('Evaluation execution finished', { pastExecutionId }); + + // Extract the output of the last node executed in the evaluation workflow + const addedMetrics = metrics.addResults(this.extractEvaluationResult(evalExecution)); + + if (evalExecution.data.resultData.error) { + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); + } else { + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementPassed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsCompleted( + testRun.id, + pastExecutionId, + addedMetrics, + trx, + ); + }); + } + } catch (e) { + // In case of an unexpected error, increment the failed count and continue with the next test case + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); + + this.errorReporter.error(e); } - } catch (e) { - // In case of an unexpected error, increment the failed count and continue with the next test case - await this.testRunRepository.incrementFailed(testRun.id); - - this.errorReporter.error(e); } + + // Mark the test run as completed or cancelled + if (abortSignal.aborted) { + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRun.id, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); + }); + } else { + const aggregatedMetrics = metrics.getAggregatedMetrics(); + await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics); + + this.logger.debug('Test run finished', { testId: test.id }); + } + } catch (e) { + if (e instanceof ExecutionCancelledError) { + this.logger.debug('Evaluation execution was cancelled. Cancelling test run', { + testRunId: testRun.id, + stoppedOn: e.extra?.executionId, + }); + + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRun.id, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); + }); + } else { + throw e; + } + } finally { + // Clean up abort controller + this.abortControllers.delete(testRun.id); } + } - const aggregatedMetrics = metrics.getAggregatedMetrics(); + /** + * Checks if the test run in a cancellable state. + */ + canBeCancelled(testRun: TestRun) { + return testRun.status !== 'running' && testRun.status !== 'new'; + } - await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics); + /** + * Cancels the test run with the given ID. + * TODO: Implement the cancellation of the test run in a multi-main scenario + */ + async cancelTestRun(testRunId: string) { + const abortController = this.abortControllers.get(testRunId); + if (abortController) { + abortController.abort(); + this.abortControllers.delete(testRunId); + } else { + // If there is no abort controller - just mark the test run and all its' pending test case executions as cancelled + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRunId, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRunId, trx); + }); + } } } diff --git a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts index aae71376e4..61853e6e42 100644 --- a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts @@ -1,7 +1,14 @@ +import express from 'express'; +import { InstanceSettings } from 'n8n-core'; + +import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; -import { Delete, Get, RestController } from '@/decorators'; +import { Delete, Get, Post, RestController } from '@/decorators'; +import { ConflictError } from '@/errors/response-errors/conflict.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import { NotImplementedError } from '@/errors/response-errors/not-implemented.error'; import { TestRunsRequest } from '@/evaluation.ee/test-definitions.types.ee'; +import { TestRunnerService } from '@/evaluation.ee/test-runner/test-runner.service.ee'; import { listQueryMiddleware } from '@/middlewares'; import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service'; @@ -12,9 +19,13 @@ export class TestRunsController { constructor( private readonly testDefinitionService: TestDefinitionService, private readonly testRunRepository: TestRunRepository, + private readonly testCaseExecutionRepository: TestCaseExecutionRepository, + private readonly testRunnerService: TestRunnerService, + private readonly instanceSettings: InstanceSettings, ) {} - /** This method is used in multiple places in the controller to get the test definition + /** + * This method is used in multiple places in the controller to get the test definition * (or just check that it exists and the user has access to it). */ private async getTestDefinition( @@ -34,6 +45,23 @@ export class TestRunsController { return testDefinition; } + /** + * Get the test run (or just check that it exists and the user has access to it) + */ + private async getTestRun( + req: TestRunsRequest.GetOne | TestRunsRequest.Delete | TestRunsRequest.Cancel, + ) { + const { id: testRunId, testDefinitionId } = req.params; + + const testRun = await this.testRunRepository.findOne({ + where: { id: testRunId, testDefinition: { id: testDefinitionId } }, + }); + + if (!testRun) throw new NotFoundError('Test run not found'); + + return testRun; + } + @Get('/:testDefinitionId/runs', { middlewares: listQueryMiddleware }) async getMany(req: TestRunsRequest.GetMany) { const { testDefinitionId } = req.params; @@ -45,33 +73,53 @@ export class TestRunsController { @Get('/:testDefinitionId/runs/:id') async getOne(req: TestRunsRequest.GetOne) { - const { id: testRunId, testDefinitionId } = req.params; - await this.getTestDefinition(req); - const testRun = await this.testRunRepository.findOne({ - where: { id: testRunId, testDefinition: { id: testDefinitionId } }, + return await this.getTestRun(req); + } + + @Get('/:testDefinitionId/runs/:id/cases') + async getTestCases(req: TestRunsRequest.GetCases) { + await this.getTestDefinition(req); + await this.getTestRun(req); + + return await this.testCaseExecutionRepository.find({ + where: { testRun: { id: req.params.id } }, }); - - if (!testRun) throw new NotFoundError('Test run not found'); - - return testRun; } @Delete('/:testDefinitionId/runs/:id') async delete(req: TestRunsRequest.Delete) { - const { id: testRunId, testDefinitionId } = req.params; + const { id: testRunId } = req.params; + // Check test definition and test run exist await this.getTestDefinition(req); - - const testRun = await this.testRunRepository.findOne({ - where: { id: testRunId, testDefinition: { id: testDefinitionId } }, - }); - - if (!testRun) throw new NotFoundError('Test run not found'); + await this.getTestRun(req); await this.testRunRepository.delete({ id: testRunId }); return { success: true }; } + + @Post('/:testDefinitionId/runs/:id/cancel') + async cancel(req: TestRunsRequest.Cancel, res: express.Response) { + if (this.instanceSettings.isMultiMain) { + throw new NotImplementedError('Cancelling test runs is not yet supported in multi-main mode'); + } + + const { id: testRunId } = req.params; + + // Check test definition and test run exist + await this.getTestDefinition(req); + const testRun = await this.getTestRun(req); + + if (this.testRunnerService.canBeCancelled(testRun)) { + const message = `The test run "${testRunId}" cannot be cancelled`; + throw new ConflictError(message); + } + + await this.testRunnerService.cancelTestRun(testRunId); + + res.status(202).json({ success: true }); + } } diff --git a/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts index 4727c8ef72..1729509210 100644 --- a/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts @@ -934,6 +934,7 @@ describe('LogStreamingEventRelay', () => { it('should log on `execution-throttled` event', () => { const event: RelayEventMap['execution-throttled'] = { executionId: 'exec123456', + type: 'production', }; eventService.emit('execution-throttled', event); @@ -942,6 +943,7 @@ describe('LogStreamingEventRelay', () => { eventName: 'n8n.execution.throttled', payload: { executionId: 'exec123456', + type: 'production', }, }); }); diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index 01673cd375..8f73b475ca 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -1155,22 +1155,6 @@ describe('TelemetryEventRelay', () => { expect(telemetry.trackWorkflowExecution).not.toHaveBeenCalled(); }); - it('should not track when execution status is "waiting"', async () => { - const event: RelayEventMap['workflow-post-execute'] = { - workflow: mockWorkflowBase, - executionId: 'execution123', - userId: 'user123', - runData: { - status: 'waiting', - data: { resultData: {} }, - } as IRun, - }; - - eventService.emit('workflow-post-execute', event); - - expect(telemetry.trackWorkflowExecution).not.toHaveBeenCalled(); - }); - it('should track successful workflow execution', async () => { const runData = mock({ finished: true, diff --git a/packages/cli/src/events/maps/relay.event-map.ts b/packages/cli/src/events/maps/relay.event-map.ts index 0e72564571..b174c66587 100644 --- a/packages/cli/src/events/maps/relay.event-map.ts +++ b/packages/cli/src/events/maps/relay.event-map.ts @@ -1,4 +1,4 @@ -import type { AuthenticationMethod } from '@n8n/api-types'; +import type { AuthenticationMethod, ProjectRelation } from '@n8n/api-types'; import type { IPersonalizationSurveyAnswersV4, IRun, @@ -6,8 +6,8 @@ import type { IWorkflowExecutionDataProcess, } from 'n8n-workflow'; +import type { ConcurrencyQueueType } from '@/concurrency/concurrency-control.service'; import type { AuthProviderType } from '@/databases/entities/auth-identity'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { GlobalRole, User } from '@/databases/entities/user'; import type { IWorkflowDb } from '@/interfaces'; @@ -338,6 +338,7 @@ export type RelayEventMap = { 'execution-throttled': { executionId: string; + type: ConcurrencyQueueType; }; 'execution-started-during-bootup': { @@ -351,10 +352,7 @@ export type RelayEventMap = { 'team-project-updated': { userId: string; role: GlobalRole; - members: Array<{ - userId: string; - role: ProjectRole; - }>; + members: ProjectRelation[]; projectId: string; }; diff --git a/packages/cli/src/events/relays/log-streaming.event-relay.ts b/packages/cli/src/events/relays/log-streaming.event-relay.ts index b048b09a83..76b578451d 100644 --- a/packages/cli/src/events/relays/log-streaming.event-relay.ts +++ b/packages/cli/src/events/relays/log-streaming.event-relay.ts @@ -385,10 +385,10 @@ export class LogStreamingEventRelay extends EventRelay { // #region Execution - private executionThrottled({ executionId }: RelayEventMap['execution-throttled']) { + private executionThrottled({ executionId, type }: RelayEventMap['execution-throttled']) { void this.eventBus.sendExecutionEvent({ eventName: 'n8n.execution.throttled', - payload: { executionId }, + payload: { executionId, type }, }); } diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index 67fbacb107..f8102f70ae 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -15,7 +15,7 @@ import { SharedWorkflowRepository } from '@/databases/repositories/shared-workfl import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { EventService } from '@/events/event.service'; import type { RelayEventMap } from '@/events/maps/relay.event-map'; -import { determineFinalExecutionStatus } from '@/execution-lifecycle-hooks/shared/shared-hook-functions'; +import { determineFinalExecutionStatus } from '@/execution-lifecycle/shared/shared-hook-functions'; import type { IExecutionTrackProperties } from '@/interfaces'; import { License } from '@/license'; import { NodeTypes } from '@/node-types'; @@ -600,11 +600,6 @@ export class TelemetryEventRelay extends EventRelay { return; } - if (runData?.status === 'waiting') { - // No need to send telemetry or logs when the workflow hasn't finished yet. - return; - } - const telemetryProperties: IExecutionTrackProperties = { workflow_id: workflow.id, is_manual: false, @@ -701,6 +696,7 @@ export class TelemetryEventRelay extends EventRelay { sharing_role: userRole, credential_type: null, is_managed: false, + ...TelemetryHelpers.resolveAIMetrics(workflow.nodes, this.nodeTypes), }; if (!manualExecEventProperties.node_graph_string) { diff --git a/packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts b/packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts deleted file mode 100644 index ac52cf3920..0000000000 --- a/packages/cli/src/execution-lifecycle-hooks/__tests__/save-execution-progress.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { ErrorReporter } from 'n8n-core'; -import { Logger } from 'n8n-core'; -import type { IRunExecutionData, ITaskData, IWorkflowBase } from 'n8n-workflow'; - -import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { saveExecutionProgress } from '@/execution-lifecycle-hooks/save-execution-progress'; -import * as fnModule from '@/execution-lifecycle-hooks/to-save-settings'; -import type { IExecutionResponse } from '@/interfaces'; -import { mockInstance } from '@test/mocking'; - -mockInstance(Logger); -const errorReporter = mockInstance(ErrorReporter); -const executionRepository = mockInstance(ExecutionRepository); - -afterEach(() => { - jest.clearAllMocks(); -}); - -const commonArgs: [IWorkflowBase, string, string, ITaskData, IRunExecutionData, string] = [ - {} as IWorkflowBase, - 'some-execution-id', - 'My Node', - {} as ITaskData, - {} as IRunExecutionData, - 'some-session-id', -]; - -const commonSettings = { error: true, success: true, manual: true }; - -test('should ignore if save settings say so', async () => { - jest.spyOn(fnModule, 'toSaveSettings').mockReturnValue({ - ...commonSettings, - progress: false, - }); - - await saveExecutionProgress(...commonArgs); - - expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); -}); - -test('should ignore on leftover async call', async () => { - jest.spyOn(fnModule, 'toSaveSettings').mockReturnValue({ - ...commonSettings, - progress: true, - }); - - executionRepository.findSingleExecution.mockResolvedValue({ - finished: true, - } as IExecutionResponse); - - await saveExecutionProgress(...commonArgs); - - expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); -}); - -test('should update execution when saving progress is enabled', async () => { - jest.spyOn(fnModule, 'toSaveSettings').mockReturnValue({ - ...commonSettings, - progress: true, - }); - - executionRepository.findSingleExecution.mockResolvedValue({} as IExecutionResponse); - - await saveExecutionProgress(...commonArgs); - - expect(executionRepository.updateExistingExecution).toHaveBeenCalledWith('some-execution-id', { - data: { - executionData: undefined, - resultData: { - lastNodeExecuted: 'My Node', - runData: { - 'My Node': [{}], - }, - }, - startData: {}, - }, - status: 'running', - }); - - expect(errorReporter.error).not.toHaveBeenCalled(); -}); - -test('should report error on failure', async () => { - jest.spyOn(fnModule, 'toSaveSettings').mockReturnValue({ - ...commonSettings, - progress: true, - }); - - const error = new Error('Something went wrong'); - - executionRepository.findSingleExecution.mockImplementation(() => { - throw error; - }); - - await saveExecutionProgress(...commonArgs); - - expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); - expect(errorReporter.error).toHaveBeenCalledWith(error); -}); diff --git a/packages/cli/src/execution-lifecycle/__tests__/execution-lifecycle-hooks.test.ts b/packages/cli/src/execution-lifecycle/__tests__/execution-lifecycle-hooks.test.ts new file mode 100644 index 0000000000..747cd775c0 --- /dev/null +++ b/packages/cli/src/execution-lifecycle/__tests__/execution-lifecycle-hooks.test.ts @@ -0,0 +1,738 @@ +import { stringify } from 'flatted'; +import { mock } from 'jest-mock-extended'; +import { + BinaryDataService, + ErrorReporter, + InstanceSettings, + Logger, + ExecutionLifecycleHooks, +} from 'n8n-core'; +import { ExpressionError } from 'n8n-workflow'; +import type { + IRunExecutionData, + ITaskData, + Workflow, + IDataObject, + IRun, + INode, + IWorkflowBase, + WorkflowExecuteMode, +} from 'n8n-workflow'; + +import config from '@/config'; +import type { Project } from '@/databases/entities/project'; +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { EventService } from '@/events/event.service'; +import { ExternalHooks } from '@/external-hooks'; +import { Push } from '@/push'; +import { OwnershipService } from '@/services/ownership.service'; +import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; +import { WorkflowExecutionService } from '@/workflows/workflow-execution.service'; +import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; +import { mockInstance } from '@test/mocking'; + +import { + getLifecycleHooksForSubExecutions, + getLifecycleHooksForRegularMain, + getLifecycleHooksForScalingWorker, + getLifecycleHooksForScalingMain, +} from '../execution-lifecycle-hooks'; + +describe('Execution Lifecycle Hooks', () => { + mockInstance(Logger); + mockInstance(InstanceSettings); + const errorReporter = mockInstance(ErrorReporter); + const eventService = mockInstance(EventService); + const executionRepository = mockInstance(ExecutionRepository); + const externalHooks = mockInstance(ExternalHooks); + const push = mockInstance(Push); + const workflowStaticDataService = mockInstance(WorkflowStaticDataService); + const workflowStatisticsService = mockInstance(WorkflowStatisticsService); + const binaryDataService = mockInstance(BinaryDataService); + const ownershipService = mockInstance(OwnershipService); + const workflowExecutionService = mockInstance(WorkflowExecutionService); + + const nodeName = 'Test Node'; + const node = mock(); + const workflowId = 'test-workflow-id'; + const executionId = 'test-execution-id'; + const workflowData: IWorkflowBase = { + id: workflowId, + name: 'Test Workflow', + active: true, + connections: {}, + nodes: [], + settings: {}, + createdAt: new Date(), + updatedAt: new Date(), + }; + const workflow = mock(); + const staticData = mock(); + const taskData = mock(); + const runExecutionData = mock(); + const successfulRun = mock({ + status: 'success', + finished: true, + waitTill: undefined, + }); + const failedRun = mock({ + status: 'error', + finished: true, + waitTill: undefined, + }); + const waitingRun = mock({ + finished: true, + status: 'waiting', + waitTill: new Date(), + }); + const expressionError = new ExpressionError('Error'); + const pushRef = 'test-push-ref'; + const retryOf = 'test-retry-of'; + + const now = new Date('2025-01-13T18:25:50.267Z'); + jest.useFakeTimers({ now }); + + let lifecycleHooks: ExecutionLifecycleHooks; + + beforeEach(() => { + jest.clearAllMocks(); + workflowData.settings = {}; + successfulRun.data = { + resultData: { + runData: {}, + }, + }; + failedRun.data = { + resultData: { + runData: {}, + error: expressionError, + }, + }; + }); + + const workflowEventTests = () => { + describe('workflowExecuteBefore', () => { + it('should emit workflow-pre-execute events', async () => { + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + + expect(eventService.emit).toHaveBeenCalledWith('workflow-pre-execute', { + executionId, + data: workflowData, + }); + }); + }); + + describe('workflowExecuteAfter', () => { + it('should emit workflow-post-execute events', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(eventService.emit).toHaveBeenCalledWith('workflow-post-execute', { + executionId, + runData: successfulRun, + workflow: workflowData, + }); + }); + + it('should not emit workflow-post-execute events for waiting executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [waitingRun, {}]); + + expect(eventService.emit).not.toHaveBeenCalledWith('workflow-post-execute'); + }); + }); + }; + + const nodeEventsTests = () => { + describe('nodeExecuteBefore', () => { + it('should emit node-pre-execute event', async () => { + await lifecycleHooks.runHook('nodeExecuteBefore', [nodeName]); + + expect(eventService.emit).toHaveBeenCalledWith('node-pre-execute', { + executionId, + workflow: workflowData, + nodeName, + }); + }); + }); + + describe('nodeExecuteAfter', () => { + it('should emit node-post-execute event', async () => { + await lifecycleHooks.runHook('nodeExecuteAfter', [nodeName, taskData, runExecutionData]); + + expect(eventService.emit).toHaveBeenCalledWith('node-post-execute', { + executionId, + workflow: workflowData, + nodeName, + }); + }); + }); + }; + + const externalHooksTests = () => { + describe('workflowExecuteBefore', () => { + it('should run workflow.preExecute hook', async () => { + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + + expect(externalHooks.run).toHaveBeenCalledWith('workflow.preExecute', [workflow, 'manual']); + }); + }); + + describe('workflowExecuteAfter', () => { + it('should run workflow.postExecute hook', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(externalHooks.run).toHaveBeenCalledWith('workflow.postExecute', [ + successfulRun, + workflowData, + executionId, + ]); + }); + }); + }; + + const statisticsTests = () => { + describe('statistics events', () => { + it('workflowExecuteAfter should emit workflowExecutionCompleted statistics event', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(workflowStatisticsService.emit).toHaveBeenCalledWith('workflowExecutionCompleted', { + workflowData, + fullRunData: successfulRun, + }); + }); + + it('nodeFetchedData should handle nodeFetchedData statistics event', async () => { + await lifecycleHooks.runHook('nodeFetchedData', [workflowId, node]); + + expect(workflowStatisticsService.emit).toHaveBeenCalledWith('nodeFetchedData', { + workflowId, + node, + }); + }); + }); + }; + + describe('getLifecycleHooksForRegularMain', () => { + const createHooks = (executionMode: WorkflowExecuteMode = 'manual') => + getLifecycleHooksForRegularMain( + { executionMode, workflowData, pushRef, retryOf }, + executionId, + ); + + beforeEach(() => { + lifecycleHooks = createHooks(); + }); + + workflowEventTests(); + nodeEventsTests(); + externalHooksTests(); + statisticsTests(); + + it('should setup the correct set of hooks', () => { + expect(lifecycleHooks).toBeInstanceOf(ExecutionLifecycleHooks); + expect(lifecycleHooks.mode).toBe('manual'); + expect(lifecycleHooks.executionId).toBe(executionId); + expect(lifecycleHooks.workflowData).toEqual(workflowData); + + const { handlers } = lifecycleHooks; + expect(handlers.nodeExecuteBefore).toHaveLength(2); + expect(handlers.nodeExecuteAfter).toHaveLength(2); + expect(handlers.workflowExecuteBefore).toHaveLength(3); + expect(handlers.workflowExecuteAfter).toHaveLength(5); + expect(handlers.nodeFetchedData).toHaveLength(1); + expect(handlers.sendResponse).toHaveLength(0); + }); + + describe('nodeExecuteBefore', () => { + it('should send nodeExecuteBefore push event', async () => { + await lifecycleHooks.runHook('nodeExecuteBefore', [nodeName]); + + expect(push.send).toHaveBeenCalledWith( + { type: 'nodeExecuteBefore', data: { executionId, nodeName } }, + pushRef, + ); + }); + }); + + describe('nodeExecuteAfter', () => { + it('should send nodeExecuteAfter push event', async () => { + await lifecycleHooks.runHook('nodeExecuteAfter', [nodeName, taskData, runExecutionData]); + + expect(push.send).toHaveBeenCalledWith( + { type: 'nodeExecuteAfter', data: { executionId, nodeName, data: taskData } }, + pushRef, + ); + }); + + it('should save execution progress when enabled', async () => { + workflowData.settings = { saveExecutionProgress: true }; + lifecycleHooks = createHooks(); + + expect(lifecycleHooks.handlers.nodeExecuteAfter).toHaveLength(3); + + await lifecycleHooks.runHook('nodeExecuteAfter', [nodeName, taskData, runExecutionData]); + + expect(executionRepository.findSingleExecution).toHaveBeenCalledWith(executionId, { + includeData: true, + unflattenData: true, + }); + }); + + it('should not save execution progress when disabled', async () => { + workflowData.settings = { saveExecutionProgress: false }; + lifecycleHooks = createHooks(); + + expect(lifecycleHooks.handlers.nodeExecuteAfter).toHaveLength(2); + + await lifecycleHooks.runHook('nodeExecuteAfter', [nodeName, taskData, runExecutionData]); + + expect(executionRepository.findSingleExecution).not.toHaveBeenCalled(); + }); + }); + + describe('workflowExecuteBefore', () => { + it('should send executionStarted push event', async () => { + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + + expect(push.send).toHaveBeenCalledWith( + { + type: 'executionStarted', + data: { + executionId, + mode: 'manual', + retryOf, + workflowId: 'test-workflow-id', + workflowName: 'Test Workflow', + startedAt: now, + flattedRunData: '[{}]', + }, + }, + pushRef, + ); + }); + + it('should run workflow.preExecute external hook', async () => { + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + + expect(externalHooks.run).toHaveBeenCalledWith('workflow.preExecute', [workflow, 'manual']); + }); + }); + + describe('workflowExecuteAfter', () => { + it('should send executionFinished push event', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + expect(push.send).toHaveBeenCalledWith( + { + type: 'executionFinished', + data: { + executionId, + rawData: stringify(successfulRun.data), + status: 'success', + workflowId: 'test-workflow-id', + }, + }, + pushRef, + ); + }); + + it('should send executionWaiting push event', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [waitingRun, {}]); + + expect(push.send).toHaveBeenCalledWith( + { + type: 'executionWaiting', + data: { executionId }, + }, + pushRef, + ); + }); + + describe('saving static data', () => { + it('should skip saving static data for manual executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(workflowStaticDataService.saveStaticDataById).not.toHaveBeenCalled(); + }); + + it('should save static data for prod executions', async () => { + lifecycleHooks = createHooks('trigger'); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(workflowStaticDataService.saveStaticDataById).toHaveBeenCalledWith( + workflowId, + staticData, + ); + }); + + it('should handle static data saving errors', async () => { + lifecycleHooks = createHooks('trigger'); + + const error = new Error('Static data save failed'); + workflowStaticDataService.saveStaticDataById.mockRejectedValueOnce(error); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(errorReporter.error).toHaveBeenCalledWith(error); + }); + }); + + describe('saving execution data', () => { + it('should update execution with proper data', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(executionRepository.updateExistingExecution).toHaveBeenCalledWith( + executionId, + expect.objectContaining({ + finished: true, + status: 'success', + }), + ); + }); + + it('should not delete unfinished executions', async () => { + const unfinishedRun = mock({ finished: false, status: 'running' }); + + await lifecycleHooks.runHook('workflowExecuteAfter', [unfinishedRun, {}]); + + expect(executionRepository.hardDelete).not.toHaveBeenCalled(); + }); + + it('should not delete waiting executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [waitingRun, {}]); + + expect(executionRepository.hardDelete).not.toHaveBeenCalled(); + }); + + it('should soft delete manual executions when manual saving is disabled', async () => { + lifecycleHooks.workflowData.settings = { saveManualExecutions: false }; + lifecycleHooks = createHooks(); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(executionRepository.softDelete).toHaveBeenCalledWith(executionId); + }); + + it('should not soft delete manual executions with waitTill', async () => { + lifecycleHooks.workflowData.settings = { saveManualExecutions: false }; + lifecycleHooks = createHooks(); + + await lifecycleHooks.runHook('workflowExecuteAfter', [waitingRun, {}]); + + expect(executionRepository.softDelete).not.toHaveBeenCalled(); + }); + }); + + describe('error workflow', () => { + it('should not execute error workflow for manual executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [failedRun, {}]); + + expect(workflowExecutionService.executeErrorWorkflow).not.toHaveBeenCalled(); + }); + + it('should execute error workflow for failed non-manual executions', async () => { + lifecycleHooks = createHooks('trigger'); + + const errorWorkflow = 'error-workflow-id'; + workflowData.settings = { errorWorkflow }; + const project = mock(); + ownershipService.getWorkflowProjectCached + .calledWith(workflowId) + .mockResolvedValue(project); + + await lifecycleHooks.runHook('workflowExecuteAfter', [failedRun, {}]); + + expect(workflowExecutionService.executeErrorWorkflow).toHaveBeenCalledWith( + errorWorkflow, + { + workflow: { + id: workflowId, + name: workflowData.name, + }, + execution: { + id: executionId, + error: expressionError, + mode: 'trigger', + retryOf, + lastNodeExecuted: undefined, + url: `http://localhost:5678/workflow/${workflowId}/executions/${executionId}`, + }, + }, + project, + ); + }); + }); + + it('should restore binary data IDs after workflow execution for webhooks', async () => { + config.set('binaryDataManager.mode', 'filesystem'); + lifecycleHooks = createHooks('webhook'); + + (successfulRun.data.resultData.runData = { + [nodeName]: [ + { + executionTime: 1, + startTime: 1, + source: [], + data: { + main: [ + [ + { + json: {}, + binary: { + data: { + id: `filesystem-v2:workflows/${workflowId}/executions/temp/binary_data/123`, + data: '', + mimeType: 'text/plain', + }, + }, + }, + ], + ], + }, + }, + ], + }), + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(binaryDataService.rename).toHaveBeenCalledWith( + 'workflows/test-workflow-id/executions/temp/binary_data/123', + 'workflows/test-workflow-id/executions/test-execution-id/binary_data/123', + ); + }); + }); + + describe("when pushRef isn't set", () => { + beforeEach(() => { + lifecycleHooks = getLifecycleHooksForRegularMain( + { executionMode: 'manual', workflowData, retryOf }, + executionId, + ); + }); + + it('should not setup any push hooks', async () => { + const { handlers } = lifecycleHooks; + expect(handlers.nodeExecuteBefore).toHaveLength(1); + expect(handlers.nodeExecuteAfter).toHaveLength(1); + expect(handlers.workflowExecuteBefore).toHaveLength(2); + expect(handlers.workflowExecuteAfter).toHaveLength(4); + + await lifecycleHooks.runHook('nodeExecuteBefore', [nodeName]); + await lifecycleHooks.runHook('nodeExecuteAfter', [nodeName, taskData, runExecutionData]); + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(push.send).not.toHaveBeenCalled(); + }); + }); + }); + + describe('getLifecycleHooksForScalingMain', () => { + beforeEach(() => { + lifecycleHooks = getLifecycleHooksForScalingMain('manual', executionId, workflowData, { + pushRef, + retryOf, + }); + }); + + workflowEventTests(); + externalHooksTests(); + + it('should setup the correct set of hooks', () => { + expect(lifecycleHooks).toBeInstanceOf(ExecutionLifecycleHooks); + expect(lifecycleHooks.mode).toBe('manual'); + expect(lifecycleHooks.executionId).toBe(executionId); + expect(lifecycleHooks.workflowData).toEqual(workflowData); + + const { handlers } = lifecycleHooks; + expect(handlers.nodeExecuteBefore).toHaveLength(0); + expect(handlers.nodeExecuteAfter).toHaveLength(0); + expect(handlers.workflowExecuteBefore).toHaveLength(2); + expect(handlers.workflowExecuteAfter).toHaveLength(4); + expect(handlers.nodeFetchedData).toHaveLength(0); + expect(handlers.sendResponse).toHaveLength(0); + }); + + describe('workflowExecuteBefore', () => { + it('should run the workflow.preExecute external hook', async () => { + await lifecycleHooks.runHook('workflowExecuteBefore', [workflow, runExecutionData]); + + expect(externalHooks.run).toHaveBeenCalledWith('workflow.preExecute', [workflow, 'manual']); + }); + }); + + describe('workflowExecuteAfter', () => { + it('should delete successful executions when success saving is disabled', async () => { + workflowData.settings = { + saveDataSuccessExecution: 'none', + saveDataErrorExecution: 'all', + }; + const lifecycleHooks = getLifecycleHooksForScalingMain( + 'webhook', + executionId, + workflowData, + { + pushRef, + retryOf, + }, + ); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, {}]); + + expect(executionRepository.hardDelete).toHaveBeenCalledWith({ + workflowId, + executionId, + }); + }); + + it('should delete failed executions when error saving is disabled', async () => { + workflowData.settings = { + saveDataSuccessExecution: 'all', + saveDataErrorExecution: 'none', + }; + const lifecycleHooks = getLifecycleHooksForScalingMain( + 'webhook', + executionId, + workflowData, + { + pushRef, + retryOf, + }, + ); + + await lifecycleHooks.runHook('workflowExecuteAfter', [failedRun, {}]); + + expect(executionRepository.hardDelete).toHaveBeenCalledWith({ + workflowId, + executionId, + }); + }); + }); + }); + + describe('getLifecycleHooksForScalingWorker', () => { + const createHooks = (executionMode: WorkflowExecuteMode = 'manual') => + getLifecycleHooksForScalingWorker(executionMode, executionId, workflowData, { + pushRef, + retryOf, + }); + + beforeEach(() => { + lifecycleHooks = createHooks(); + }); + + nodeEventsTests(); + externalHooksTests(); + statisticsTests(); + + it('should setup the correct set of hooks', () => { + expect(lifecycleHooks).toBeInstanceOf(ExecutionLifecycleHooks); + expect(lifecycleHooks.mode).toBe('manual'); + expect(lifecycleHooks.executionId).toBe(executionId); + expect(lifecycleHooks.workflowData).toEqual(workflowData); + + const { handlers } = lifecycleHooks; + expect(handlers.nodeExecuteBefore).toHaveLength(2); + expect(handlers.nodeExecuteAfter).toHaveLength(2); + expect(handlers.workflowExecuteBefore).toHaveLength(2); + expect(handlers.workflowExecuteAfter).toHaveLength(4); + expect(handlers.nodeFetchedData).toHaveLength(1); + expect(handlers.sendResponse).toHaveLength(0); + }); + + describe('saving static data', () => { + it('should skip saving static data for manual executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(workflowStaticDataService.saveStaticDataById).not.toHaveBeenCalled(); + }); + + it('should save static data for prod executions', async () => { + lifecycleHooks = createHooks('trigger'); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(workflowStaticDataService.saveStaticDataById).toHaveBeenCalledWith( + workflowId, + staticData, + ); + }); + + it('should handle static data saving errors', async () => { + lifecycleHooks = createHooks('trigger'); + const error = new Error('Static data save failed'); + workflowStaticDataService.saveStaticDataById.mockRejectedValueOnce(error); + + await lifecycleHooks.runHook('workflowExecuteAfter', [successfulRun, staticData]); + + expect(errorReporter.error).toHaveBeenCalledWith(error); + }); + }); + + describe('error workflow', () => { + it('should not execute error workflow for manual executions', async () => { + await lifecycleHooks.runHook('workflowExecuteAfter', [failedRun, {}]); + + expect(workflowExecutionService.executeErrorWorkflow).not.toHaveBeenCalled(); + }); + + it('should execute error workflow for failed non-manual executions', async () => { + lifecycleHooks = createHooks('trigger'); + const errorWorkflow = 'error-workflow-id'; + workflowData.settings = { errorWorkflow }; + const project = mock(); + ownershipService.getWorkflowProjectCached.calledWith(workflowId).mockResolvedValue(project); + + await lifecycleHooks.runHook('workflowExecuteAfter', [failedRun, {}]); + + expect(workflowExecutionService.executeErrorWorkflow).toHaveBeenCalledWith( + errorWorkflow, + { + workflow: { + id: workflowId, + name: workflowData.name, + }, + execution: { + id: executionId, + error: expressionError, + mode: 'trigger', + retryOf, + lastNodeExecuted: undefined, + url: `http://localhost:5678/workflow/${workflowId}/executions/${executionId}`, + }, + }, + project, + ); + }); + }); + }); + + describe('getLifecycleHooksForSubExecutions', () => { + beforeEach(() => { + lifecycleHooks = getLifecycleHooksForSubExecutions( + 'manual', + executionId, + workflowData, + undefined, + ); + }); + + workflowEventTests(); + nodeEventsTests(); + externalHooksTests(); + statisticsTests(); + + it('should setup the correct set of hooks', () => { + expect(lifecycleHooks).toBeInstanceOf(ExecutionLifecycleHooks); + expect(lifecycleHooks.mode).toBe('manual'); + expect(lifecycleHooks.executionId).toBe(executionId); + expect(lifecycleHooks.workflowData).toEqual(workflowData); + + const { handlers } = lifecycleHooks; + expect(handlers.nodeExecuteBefore).toHaveLength(1); + expect(handlers.nodeExecuteAfter).toHaveLength(1); + expect(handlers.workflowExecuteBefore).toHaveLength(2); + expect(handlers.workflowExecuteAfter).toHaveLength(4); + expect(handlers.nodeFetchedData).toHaveLength(1); + expect(handlers.sendResponse).toHaveLength(0); + }); + }); +}); diff --git a/packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts b/packages/cli/src/execution-lifecycle/__tests__/restore-binary-data-id.test.ts similarity index 98% rename from packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts rename to packages/cli/src/execution-lifecycle/__tests__/restore-binary-data-id.test.ts index f4f7a463bc..76ac0d4e21 100644 --- a/packages/cli/src/execution-lifecycle-hooks/__tests__/restore-binary-data-id.test.ts +++ b/packages/cli/src/execution-lifecycle/__tests__/restore-binary-data-id.test.ts @@ -2,7 +2,7 @@ import { BinaryDataService } from 'n8n-core'; import type { IRun } from 'n8n-workflow'; import config from '@/config'; -import { restoreBinaryDataId } from '@/execution-lifecycle-hooks/restore-binary-data-id'; +import { restoreBinaryDataId } from '@/execution-lifecycle/restore-binary-data-id'; import { mockInstance } from '@test/mocking'; function toIRun(item?: object) { diff --git a/packages/cli/src/execution-lifecycle/__tests__/save-execution-progress.test.ts b/packages/cli/src/execution-lifecycle/__tests__/save-execution-progress.test.ts new file mode 100644 index 0000000000..8c90c865fd --- /dev/null +++ b/packages/cli/src/execution-lifecycle/__tests__/save-execution-progress.test.ts @@ -0,0 +1,152 @@ +import { mock } from 'jest-mock-extended'; +import { ErrorReporter } from 'n8n-core'; +import { Logger } from 'n8n-core'; +import type { IRunExecutionData, ITaskData } from 'n8n-workflow'; + +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { IExecutionResponse } from '@/interfaces'; +import { mockInstance } from '@test/mocking'; + +import { saveExecutionProgress } from '../save-execution-progress'; + +describe('saveExecutionProgress', () => { + mockInstance(Logger); + const errorReporter = mockInstance(ErrorReporter); + const executionRepository = mockInstance(ExecutionRepository); + + afterEach(() => { + jest.resetAllMocks(); + }); + + const workflowId = 'some-workflow-id'; + const executionId = 'some-execution-id'; + const nodeName = 'My Node'; + const taskData = mock(); + const runExecutionData = mock(); + + const commonArgs = [workflowId, executionId, nodeName, taskData, runExecutionData] as const; + + test('should not try to update non-existent executions', async () => { + executionRepository.findSingleExecution.mockResolvedValue(undefined); + + await saveExecutionProgress(...commonArgs); + expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); + }); + + test('should handle DB errors on execution lookup', async () => { + const error = new Error('Something went wrong'); + executionRepository.findSingleExecution.mockImplementation(() => { + throw error; + }); + + await saveExecutionProgress(...commonArgs); + + expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); + expect(errorReporter.error).toHaveBeenCalledWith(error); + }); + + test('should handle DB errors when updating the execution', async () => { + const error = new Error('Something went wrong'); + executionRepository.findSingleExecution.mockResolvedValue({} as IExecutionResponse); + executionRepository.updateExistingExecution.mockImplementation(() => { + throw error; + }); + + await saveExecutionProgress(...commonArgs); + + expect(executionRepository.findSingleExecution).toHaveBeenCalled(); + expect(executionRepository.updateExistingExecution).toHaveBeenCalled(); + expect(errorReporter.error).toHaveBeenCalledWith(error); + }); + + test('should not try to update finished executions', async () => { + executionRepository.findSingleExecution.mockResolvedValue( + mock({ + finished: true, + }), + ); + + await saveExecutionProgress(...commonArgs); + + expect(executionRepository.updateExistingExecution).not.toHaveBeenCalled(); + }); + + test('should populate `.data` when it is missing', async () => { + const fullExecutionData = {} as IExecutionResponse; + executionRepository.findSingleExecution.mockResolvedValue(fullExecutionData); + + await saveExecutionProgress(...commonArgs); + + expect(fullExecutionData).toEqual({ + data: { + executionData: runExecutionData.executionData, + resultData: { + lastNodeExecuted: nodeName, + runData: { + [nodeName]: [taskData], + }, + }, + startData: {}, + }, + status: 'running', + }); + + expect(executionRepository.updateExistingExecution).toHaveBeenCalledWith( + executionId, + fullExecutionData, + ); + + expect(errorReporter.error).not.toHaveBeenCalled(); + }); + + test('should augment `.data` if it already exists', async () => { + const fullExecutionData = { + data: { + startData: {}, + resultData: { + runData: { + [nodeName]: [{}], + }, + }, + }, + } as unknown as IExecutionResponse; + executionRepository.findSingleExecution.mockResolvedValue(fullExecutionData); + + await saveExecutionProgress(...commonArgs); + + expect(fullExecutionData).toEqual({ + data: { + executionData: runExecutionData.executionData, + resultData: { + lastNodeExecuted: nodeName, + runData: { + [nodeName]: [{}, taskData], + }, + }, + startData: {}, + }, + status: 'running', + }); + + expect(executionRepository.updateExistingExecution).toHaveBeenCalledWith( + executionId, + fullExecutionData, + ); + }); + + test('should set last executed node correctly', async () => { + const fullExecutionData = { + data: { + resultData: { + lastNodeExecuted: 'Another Node', + runData: {}, + }, + }, + } as unknown as IExecutionResponse; + executionRepository.findSingleExecution.mockResolvedValue(fullExecutionData); + + await saveExecutionProgress(...commonArgs); + + expect(fullExecutionData.data.resultData.lastNodeExecuted).toEqual(nodeName); + }); +}); diff --git a/packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts b/packages/cli/src/execution-lifecycle/__tests__/to-save-settings.test.ts similarity index 98% rename from packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts rename to packages/cli/src/execution-lifecycle/__tests__/to-save-settings.test.ts index f12c209827..142b3c34ce 100644 --- a/packages/cli/src/execution-lifecycle-hooks/__tests__/to-save-settings.test.ts +++ b/packages/cli/src/execution-lifecycle/__tests__/to-save-settings.test.ts @@ -1,5 +1,6 @@ import config from '@/config'; -import { toSaveSettings } from '@/execution-lifecycle-hooks/to-save-settings'; + +import { toSaveSettings } from '../to-save-settings'; afterEach(() => { config.load(config.default); diff --git a/packages/cli/src/execution-lifecycle/execute-error-workflow.ts b/packages/cli/src/execution-lifecycle/execute-error-workflow.ts new file mode 100644 index 0000000000..fefce8a97b --- /dev/null +++ b/packages/cli/src/execution-lifecycle/execute-error-workflow.ts @@ -0,0 +1,130 @@ +import { GlobalConfig } from '@n8n/config'; +import { Container } from '@n8n/di'; +import { ErrorReporter, Logger } from 'n8n-core'; +import type { IRun, IWorkflowBase, WorkflowExecuteMode } from 'n8n-workflow'; + +import type { IWorkflowErrorData } from '@/interfaces'; +import { OwnershipService } from '@/services/ownership.service'; +import { UrlService } from '@/services/url.service'; +import { WorkflowExecutionService } from '@/workflows/workflow-execution.service'; + +/** + * Checks if there was an error and if errorWorkflow or a trigger is defined. If so it collects + * all the data and executes it + * + * @param {IWorkflowBase} workflowData The workflow which got executed + * @param {IRun} fullRunData The run which produced the error + * @param {WorkflowExecuteMode} mode The mode in which the workflow got started in + * @param {string} [executionId] The id the execution got saved as + */ +export function executeErrorWorkflow( + workflowData: IWorkflowBase, + fullRunData: IRun, + mode: WorkflowExecuteMode, + executionId?: string, + retryOf?: string, +): void { + const logger = Container.get(Logger); + + // Check if there was an error and if so if an errorWorkflow or a trigger is set + let pastExecutionUrl: string | undefined; + if (executionId !== undefined) { + pastExecutionUrl = `${Container.get(UrlService).getWebhookBaseUrl()}workflow/${ + workflowData.id + }/executions/${executionId}`; + } + + if (fullRunData.data.resultData.error !== undefined) { + let workflowErrorData: IWorkflowErrorData; + const workflowId = workflowData.id; + + if (executionId) { + // The error did happen in an execution + workflowErrorData = { + execution: { + id: executionId, + url: pastExecutionUrl, + error: fullRunData.data.resultData.error, + lastNodeExecuted: fullRunData.data.resultData.lastNodeExecuted!, + mode, + retryOf, + }, + workflow: { + id: workflowId, + name: workflowData.name, + }, + }; + } else { + // The error did happen in a trigger + workflowErrorData = { + trigger: { + error: fullRunData.data.resultData.error, + mode, + }, + workflow: { + id: workflowId, + name: workflowData.name, + }, + }; + } + + const { errorTriggerType } = Container.get(GlobalConfig).nodes; + // Run the error workflow + // To avoid an infinite loop do not run the error workflow again if the error-workflow itself failed and it is its own error-workflow. + const { errorWorkflow } = workflowData.settings ?? {}; + if (errorWorkflow && !(mode === 'error' && workflowId && errorWorkflow === workflowId)) { + logger.debug('Start external error workflow', { + executionId, + errorWorkflowId: errorWorkflow, + workflowId, + }); + // If a specific error workflow is set run only that one + + // First, do permission checks. + if (!workflowId) { + // Manual executions do not trigger error workflows + // So this if should never happen. It was added to + // make sure there are no possible security gaps + return; + } + + Container.get(OwnershipService) + .getWorkflowProjectCached(workflowId) + .then((project) => { + void Container.get(WorkflowExecutionService).executeErrorWorkflow( + errorWorkflow, + workflowErrorData, + project, + ); + }) + .catch((error: Error) => { + Container.get(ErrorReporter).error(error); + logger.error( + `Could not execute ErrorWorkflow for execution ID ${executionId} because of error querying the workflow owner`, + { + executionId, + errorWorkflowId: errorWorkflow, + workflowId, + error, + workflowErrorData, + }, + ); + }); + } else if ( + mode !== 'error' && + workflowId !== undefined && + workflowData.nodes.some((node) => node.type === errorTriggerType) + ) { + logger.debug('Start internal error workflow', { executionId, workflowId }); + void Container.get(OwnershipService) + .getWorkflowProjectCached(workflowId) + .then((project) => { + void Container.get(WorkflowExecutionService).executeErrorWorkflow( + workflowId, + workflowErrorData, + project, + ); + }); + } + } +} diff --git a/packages/cli/src/execution-lifecycle/execution-lifecycle-hooks.ts b/packages/cli/src/execution-lifecycle/execution-lifecycle-hooks.ts new file mode 100644 index 0000000000..237bd25edb --- /dev/null +++ b/packages/cli/src/execution-lifecycle/execution-lifecycle-hooks.ts @@ -0,0 +1,482 @@ +import { Container } from '@n8n/di'; +import { stringify } from 'flatted'; +import { ErrorReporter, Logger, InstanceSettings, ExecutionLifecycleHooks } from 'n8n-core'; +import type { + IWorkflowBase, + WorkflowExecuteMode, + IWorkflowExecutionDataProcess, +} from 'n8n-workflow'; + +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { EventService } from '@/events/event.service'; +import { ExternalHooks } from '@/external-hooks'; +import { Push } from '@/push'; +import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; +import { isWorkflowIdValid } from '@/utils'; +import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; + +import { executeErrorWorkflow } from './execute-error-workflow'; +import { restoreBinaryDataId } from './restore-binary-data-id'; +import { saveExecutionProgress } from './save-execution-progress'; +import { + determineFinalExecutionStatus, + prepareExecutionDataForDbUpdate, + updateExistingExecution, +} from './shared/shared-hook-functions'; +import { type ExecutionSaveSettings, toSaveSettings } from './to-save-settings'; + +type HooksSetupParameters = { + saveSettings: ExecutionSaveSettings; + pushRef?: string; + retryOf?: string; +}; + +function hookFunctionsWorkflowEvents(hooks: ExecutionLifecycleHooks, userId?: string) { + const eventService = Container.get(EventService); + hooks.addHandler('workflowExecuteBefore', function () { + const { executionId, workflowData } = this; + eventService.emit('workflow-pre-execute', { executionId, data: workflowData }); + }); + hooks.addHandler('workflowExecuteAfter', function (runData) { + if (runData.status === 'waiting') return; + + const { executionId, workflowData: workflow } = this; + eventService.emit('workflow-post-execute', { executionId, runData, workflow, userId }); + }); +} + +function hookFunctionsNodeEvents(hooks: ExecutionLifecycleHooks) { + const eventService = Container.get(EventService); + hooks.addHandler('nodeExecuteBefore', function (nodeName) { + const { executionId, workflowData: workflow } = this; + eventService.emit('node-pre-execute', { executionId, workflow, nodeName }); + }); + hooks.addHandler('nodeExecuteAfter', function (nodeName) { + const { executionId, workflowData: workflow } = this; + eventService.emit('node-post-execute', { executionId, workflow, nodeName }); + }); +} + +/** + * Returns hook functions to push data to Editor-UI + */ +function hookFunctionsPush( + hooks: ExecutionLifecycleHooks, + { pushRef, retryOf }: HooksSetupParameters, +) { + if (!pushRef) return; + const logger = Container.get(Logger); + const pushInstance = Container.get(Push); + hooks.addHandler('nodeExecuteBefore', function (nodeName) { + const { executionId } = this; + // Push data to session which started workflow before each + // node which starts rendering + logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { + executionId, + pushRef, + workflowId: this.workflowData.id, + }); + + pushInstance.send({ type: 'nodeExecuteBefore', data: { executionId, nodeName } }, pushRef); + }); + hooks.addHandler('nodeExecuteAfter', function (nodeName, data) { + const { executionId } = this; + // Push data to session which started workflow after each rendered node + logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { + executionId, + pushRef, + workflowId: this.workflowData.id, + }); + + pushInstance.send({ type: 'nodeExecuteAfter', data: { executionId, nodeName, data } }, pushRef); + }); + hooks.addHandler('workflowExecuteBefore', function (_workflow, data) { + const { executionId } = this; + const { id: workflowId, name: workflowName } = this.workflowData; + logger.debug('Executing hook (hookFunctionsPush)', { + executionId, + pushRef, + workflowId, + }); + // Push data to session which started the workflow + pushInstance.send( + { + type: 'executionStarted', + data: { + executionId, + mode: this.mode, + startedAt: new Date(), + retryOf, + workflowId, + workflowName, + flattedRunData: data?.resultData.runData + ? stringify(data.resultData.runData) + : stringify({}), + }, + }, + pushRef, + ); + }); + hooks.addHandler('workflowExecuteAfter', function (fullRunData) { + const { executionId } = this; + const { id: workflowId } = this.workflowData; + logger.debug('Executing hook (hookFunctionsPush)', { + executionId, + pushRef, + workflowId, + }); + + const { status } = fullRunData; + if (status === 'waiting') { + pushInstance.send({ type: 'executionWaiting', data: { executionId } }, pushRef); + } else { + const rawData = stringify(fullRunData.data); + pushInstance.send( + { type: 'executionFinished', data: { executionId, workflowId, status, rawData } }, + pushRef, + ); + } + }); +} + +function hookFunctionsExternalHooks(hooks: ExecutionLifecycleHooks) { + const externalHooks = Container.get(ExternalHooks); + hooks.addHandler('workflowExecuteBefore', async function (workflow) { + await externalHooks.run('workflow.preExecute', [workflow, this.mode]); + }); + hooks.addHandler('workflowExecuteAfter', async function (fullRunData) { + await externalHooks.run('workflow.postExecute', [ + fullRunData, + this.workflowData, + this.executionId, + ]); + }); +} + +function hookFunctionsSaveProgress( + hooks: ExecutionLifecycleHooks, + { saveSettings }: HooksSetupParameters, +) { + if (!saveSettings.progress) return; + hooks.addHandler('nodeExecuteAfter', async function (nodeName, data, executionData) { + await saveExecutionProgress( + this.workflowData.id, + this.executionId, + nodeName, + data, + executionData, + ); + }); +} + +/** This should ideally be added before any other `workflowExecuteAfter` hook to ensure all hooks get the same execution status */ +function hookFunctionsFinalizeExecutionStatus(hooks: ExecutionLifecycleHooks) { + hooks.addHandler('workflowExecuteAfter', (fullRunData) => { + fullRunData.status = determineFinalExecutionStatus(fullRunData); + }); +} + +function hookFunctionsStatistics(hooks: ExecutionLifecycleHooks) { + const workflowStatisticsService = Container.get(WorkflowStatisticsService); + hooks.addHandler('nodeFetchedData', (workflowId, node) => { + workflowStatisticsService.emit('nodeFetchedData', { workflowId, node }); + }); +} + +/** + * Returns hook functions to save workflow execution and call error workflow + */ +function hookFunctionsSave( + hooks: ExecutionLifecycleHooks, + { pushRef, retryOf, saveSettings }: HooksSetupParameters, +) { + const logger = Container.get(Logger); + const errorReporter = Container.get(ErrorReporter); + const executionRepository = Container.get(ExecutionRepository); + const workflowStaticDataService = Container.get(WorkflowStaticDataService); + const workflowStatisticsService = Container.get(WorkflowStatisticsService); + hooks.addHandler('workflowExecuteAfter', async function (fullRunData, newStaticData) { + logger.debug('Executing hook (hookFunctionsSave)', { + executionId: this.executionId, + workflowId: this.workflowData.id, + }); + + await restoreBinaryDataId(fullRunData, this.executionId, this.mode); + + const isManualMode = this.mode === 'manual'; + + try { + if (!isManualMode && isWorkflowIdValid(this.workflowData.id) && newStaticData) { + // Workflow is saved so update in database + try { + await workflowStaticDataService.saveStaticDataById(this.workflowData.id, newStaticData); + } catch (e) { + errorReporter.error(e); + logger.error( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + `There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (hookFunctionsSave)`, + { executionId: this.executionId, workflowId: this.workflowData.id }, + ); + } + } + + if (isManualMode && !saveSettings.manual && !fullRunData.waitTill) { + /** + * When manual executions are not being saved, we only soft-delete + * the execution so that the user can access its binary data + * while building their workflow. + * + * The manual execution and its binary data will be hard-deleted + * on the next pruning cycle after the grace period set by + * `EXECUTIONS_DATA_HARD_DELETE_BUFFER`. + */ + await executionRepository.softDelete(this.executionId); + + return; + } + + const shouldNotSave = + (fullRunData.status === 'success' && !saveSettings.success) || + (fullRunData.status !== 'success' && !saveSettings.error); + + if (shouldNotSave && !fullRunData.waitTill && !isManualMode) { + executeErrorWorkflow(this.workflowData, fullRunData, this.mode, this.executionId, retryOf); + + await executionRepository.hardDelete({ + workflowId: this.workflowData.id, + executionId: this.executionId, + }); + + return; + } + + // Although it is treated as IWorkflowBase here, it's being instantiated elsewhere with properties that may be sensitive + // As a result, we should create an IWorkflowBase object with only the data we want to save in it. + const fullExecutionData = prepareExecutionDataForDbUpdate({ + runData: fullRunData, + workflowData: this.workflowData, + workflowStatusFinal: fullRunData.status, + retryOf, + }); + + // When going into the waiting state, store the pushRef in the execution-data + if (fullRunData.waitTill && isManualMode) { + fullExecutionData.data.pushRef = pushRef; + } + + await updateExistingExecution({ + executionId: this.executionId, + workflowId: this.workflowData.id, + executionData: fullExecutionData, + }); + + if (!isManualMode) { + executeErrorWorkflow(this.workflowData, fullRunData, this.mode, this.executionId, retryOf); + } + } finally { + workflowStatisticsService.emit('workflowExecutionCompleted', { + workflowData: this.workflowData, + fullRunData, + }); + } + }); +} + +/** + * Returns hook functions to save workflow execution and call error workflow + * for running with queues. Manual executions should never run on queues as + * they are always executed in the main process. + */ +function hookFunctionsSaveWorker( + hooks: ExecutionLifecycleHooks, + { pushRef, retryOf }: HooksSetupParameters, +) { + const logger = Container.get(Logger); + const errorReporter = Container.get(ErrorReporter); + const workflowStaticDataService = Container.get(WorkflowStaticDataService); + const workflowStatisticsService = Container.get(WorkflowStatisticsService); + hooks.addHandler('workflowExecuteAfter', async function (fullRunData, newStaticData) { + logger.debug('Executing hook (hookFunctionsSaveWorker)', { + executionId: this.executionId, + workflowId: this.workflowData.id, + }); + + const isManualMode = this.mode === 'manual'; + + try { + if (!isManualMode && isWorkflowIdValid(this.workflowData.id) && newStaticData) { + // Workflow is saved so update in database + try { + await workflowStaticDataService.saveStaticDataById(this.workflowData.id, newStaticData); + } catch (e) { + errorReporter.error(e); + logger.error( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + `There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (workflowExecuteAfter)`, + { workflowId: this.workflowData.id }, + ); + } + } + + if (!isManualMode && fullRunData.status !== 'success' && fullRunData.status !== 'waiting') { + executeErrorWorkflow(this.workflowData, fullRunData, this.mode, this.executionId, retryOf); + } + + // Although it is treated as IWorkflowBase here, it's being instantiated elsewhere with properties that may be sensitive + // As a result, we should create an IWorkflowBase object with only the data we want to save in it. + const fullExecutionData = prepareExecutionDataForDbUpdate({ + runData: fullRunData, + workflowData: this.workflowData, + workflowStatusFinal: fullRunData.status, + retryOf, + }); + + // When going into the waiting state, store the pushRef in the execution-data + if (fullRunData.waitTill && isManualMode) { + fullExecutionData.data.pushRef = pushRef; + } + + await updateExistingExecution({ + executionId: this.executionId, + workflowId: this.workflowData.id, + executionData: fullExecutionData, + }); + } finally { + workflowStatisticsService.emit('workflowExecutionCompleted', { + workflowData: this.workflowData, + fullRunData, + }); + } + }); +} + +/** + * Returns ExecutionLifecycleHooks instance for running integrated workflows + * (Workflows which get started inside of another workflow) + */ +export function getLifecycleHooksForSubExecutions( + mode: WorkflowExecuteMode, + executionId: string, + workflowData: IWorkflowBase, + userId?: string, +): ExecutionLifecycleHooks { + const hooks = new ExecutionLifecycleHooks(mode, executionId, workflowData); + const saveSettings = toSaveSettings(workflowData.settings); + hookFunctionsWorkflowEvents(hooks, userId); + hookFunctionsNodeEvents(hooks); + hookFunctionsFinalizeExecutionStatus(hooks); + hookFunctionsSave(hooks, { saveSettings }); + hookFunctionsSaveProgress(hooks, { saveSettings }); + hookFunctionsStatistics(hooks); + hookFunctionsExternalHooks(hooks); + return hooks; +} + +/** + * Returns ExecutionLifecycleHooks instance for worker in scaling mode. + */ +export function getLifecycleHooksForScalingWorker( + mode: WorkflowExecuteMode, + executionId: string, + workflowData: IWorkflowBase, + { pushRef, retryOf }: Omit = {}, +): ExecutionLifecycleHooks { + const hooks = new ExecutionLifecycleHooks(mode, executionId, workflowData); + const saveSettings = toSaveSettings(workflowData.settings); + const optionalParameters = { pushRef, retryOf, saveSettings }; + hookFunctionsNodeEvents(hooks); + hookFunctionsFinalizeExecutionStatus(hooks); + hookFunctionsSaveWorker(hooks, optionalParameters); + hookFunctionsSaveProgress(hooks, optionalParameters); + hookFunctionsStatistics(hooks); + hookFunctionsExternalHooks(hooks); + + if (mode === 'manual' && Container.get(InstanceSettings).isWorker) { + hookFunctionsPush(hooks, optionalParameters); + } + + return hooks; +} + +/** + * Returns ExecutionLifecycleHooks instance for main process if workflow runs via worker + */ +export function getLifecycleHooksForScalingMain( + mode: WorkflowExecuteMode, + executionId: string, + workflowData: IWorkflowBase, + { pushRef, retryOf }: Omit = {}, +): ExecutionLifecycleHooks { + const hooks = new ExecutionLifecycleHooks(mode, executionId, workflowData); + const saveSettings = toSaveSettings(workflowData.settings); + const optionalParameters = { pushRef, retryOf, saveSettings }; + const executionRepository = Container.get(ExecutionRepository); + + hookFunctionsWorkflowEvents(hooks); + hookFunctionsSaveProgress(hooks, optionalParameters); + hookFunctionsExternalHooks(hooks); + hookFunctionsFinalizeExecutionStatus(hooks); + + hooks.addHandler('workflowExecuteAfter', async function (fullRunData) { + // Don't delete executions before they are finished + if (!fullRunData.finished) return; + + const isManualMode = this.mode === 'manual'; + + if (isManualMode && !saveSettings.manual && !fullRunData.waitTill) { + /** + * When manual executions are not being saved, we only soft-delete + * the execution so that the user can access its binary data + * while building their workflow. + * + * The manual execution and its binary data will be hard-deleted + * on the next pruning cycle after the grace period set by + * `EXECUTIONS_DATA_HARD_DELETE_BUFFER`. + */ + await executionRepository.softDelete(this.executionId); + + return; + } + + const shouldNotSave = + (fullRunData.status === 'success' && !saveSettings.success) || + (fullRunData.status !== 'success' && !saveSettings.error); + + if (!isManualMode && shouldNotSave && !fullRunData.waitTill) { + await executionRepository.hardDelete({ + workflowId: this.workflowData.id, + executionId: this.executionId, + }); + } + }); + + // When running with worker mode, main process executes + // Only workflowExecuteBefore + workflowExecuteAfter + // So to avoid confusion, we are removing other hooks. + hooks.handlers.nodeExecuteBefore = []; + hooks.handlers.nodeExecuteAfter = []; + + return hooks; +} + +/** + * Returns ExecutionLifecycleHooks instance for running the main workflow + */ +export function getLifecycleHooksForRegularMain( + data: IWorkflowExecutionDataProcess, + executionId: string, +): ExecutionLifecycleHooks { + const { pushRef, retryOf, executionMode, workflowData } = data; + const hooks = new ExecutionLifecycleHooks(executionMode, executionId, workflowData); + const saveSettings = toSaveSettings(workflowData.settings); + const optionalParameters = { pushRef, retryOf: retryOf ?? undefined, saveSettings }; + hookFunctionsWorkflowEvents(hooks); + hookFunctionsNodeEvents(hooks); + hookFunctionsFinalizeExecutionStatus(hooks); + hookFunctionsSave(hooks, optionalParameters); + hookFunctionsPush(hooks, optionalParameters); + hookFunctionsSaveProgress(hooks, optionalParameters); + hookFunctionsStatistics(hooks); + hookFunctionsExternalHooks(hooks); + return hooks; +} diff --git a/packages/cli/src/execution-lifecycle-hooks/restore-binary-data-id.ts b/packages/cli/src/execution-lifecycle/restore-binary-data-id.ts similarity index 100% rename from packages/cli/src/execution-lifecycle-hooks/restore-binary-data-id.ts rename to packages/cli/src/execution-lifecycle/restore-binary-data-id.ts diff --git a/packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts b/packages/cli/src/execution-lifecycle/save-execution-progress.ts similarity index 53% rename from packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts rename to packages/cli/src/execution-lifecycle/save-execution-progress.ts index 9e751c90f6..78f61ff98f 100644 --- a/packages/cli/src/execution-lifecycle-hooks/save-execution-progress.ts +++ b/packages/cli/src/execution-lifecycle/save-execution-progress.ts @@ -1,23 +1,19 @@ import { Container } from '@n8n/di'; import { ErrorReporter, Logger } from 'n8n-core'; -import type { IRunExecutionData, ITaskData, IWorkflowBase } from 'n8n-workflow'; +import type { IRunExecutionData, ITaskData } from 'n8n-workflow'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { toSaveSettings } from '@/execution-lifecycle-hooks/to-save-settings'; export async function saveExecutionProgress( - workflowData: IWorkflowBase, + workflowId: string, executionId: string, nodeName: string, data: ITaskData, executionData: IRunExecutionData, - pushRef?: string, ) { - const saveSettings = toSaveSettings(workflowData.settings); - - if (!saveSettings.progress) return; - const logger = Container.get(Logger); + const executionRepository = Container.get(ExecutionRepository); + const errorReporter = Container.get(ErrorReporter); try { logger.debug(`Save execution progress to database for execution ID ${executionId} `, { @@ -25,13 +21,10 @@ export async function saveExecutionProgress( nodeName, }); - const fullExecutionData = await Container.get(ExecutionRepository).findSingleExecution( - executionId, - { - includeData: true, - unflattenData: true, - }, - ); + const fullExecutionData = await executionRepository.findSingleExecution(executionId, { + includeData: true, + unflattenData: true, + }); if (!fullExecutionData) { // Something went badly wrong if this happens. @@ -46,29 +39,22 @@ export async function saveExecutionProgress( return; } - if (fullExecutionData.data === undefined) { - fullExecutionData.data = { - startData: {}, - resultData: { - runData: {}, - }, - executionData: { - contextData: {}, - metadata: {}, - nodeExecutionStack: [], - waitingExecution: {}, - waitingExecutionSource: {}, - }, - }; - } + fullExecutionData.data ??= { + startData: {}, + resultData: { + runData: {}, + }, + executionData: { + contextData: {}, + metadata: {}, + nodeExecutionStack: [], + waitingExecution: {}, + waitingExecutionSource: {}, + }, + }; - if (Array.isArray(fullExecutionData.data.resultData.runData[nodeName])) { - // Append data if array exists - fullExecutionData.data.resultData.runData[nodeName].push(data); - } else { - // Initialize array and save data - fullExecutionData.data.resultData.runData[nodeName] = [data]; - } + const { runData } = fullExecutionData.data.resultData; + (runData[nodeName] ??= []).push(data); fullExecutionData.data.executionData = executionData.executionData; @@ -77,27 +63,19 @@ export async function saveExecutionProgress( fullExecutionData.status = 'running'; - await Container.get(ExecutionRepository).updateExistingExecution( - executionId, - fullExecutionData, - ); + await executionRepository.updateExistingExecution(executionId, fullExecutionData); } catch (e) { const error = e instanceof Error ? e : new Error(`${e}`); - Container.get(ErrorReporter).error(error); + errorReporter.error(error); // TODO: Improve in the future! // Errors here might happen because of database access // For busy machines, we may get "Database is locked" errors. // We do this to prevent crashes and executions ending in `unknown` state. logger.error( - `Failed saving execution progress to database for execution ID ${executionId} (hookFunctionsPreExecute, nodeExecuteAfter)`, - { - ...error, - executionId, - pushRef, - workflowId: workflowData.id, - }, + `Failed saving execution progress to database for execution ID ${executionId} (hookFunctionsSaveProgress, nodeExecuteAfter)`, + { error, executionId, workflowId }, ); } } diff --git a/packages/cli/src/execution-lifecycle-hooks/shared/__tests__/shared-hook-functions.test.ts b/packages/cli/src/execution-lifecycle/shared/__tests__/shared-hook-functions.test.ts similarity index 100% rename from packages/cli/src/execution-lifecycle-hooks/shared/__tests__/shared-hook-functions.test.ts rename to packages/cli/src/execution-lifecycle/shared/__tests__/shared-hook-functions.test.ts diff --git a/packages/cli/src/execution-lifecycle-hooks/shared/shared-hook-functions.ts b/packages/cli/src/execution-lifecycle/shared/shared-hook-functions.ts similarity index 100% rename from packages/cli/src/execution-lifecycle-hooks/shared/shared-hook-functions.ts rename to packages/cli/src/execution-lifecycle/shared/shared-hook-functions.ts diff --git a/packages/cli/src/execution-lifecycle-hooks/to-save-settings.ts b/packages/cli/src/execution-lifecycle/to-save-settings.ts similarity index 88% rename from packages/cli/src/execution-lifecycle-hooks/to-save-settings.ts rename to packages/cli/src/execution-lifecycle/to-save-settings.ts index a7af8f3ddc..b5cd24f7d9 100644 --- a/packages/cli/src/execution-lifecycle-hooks/to-save-settings.ts +++ b/packages/cli/src/execution-lifecycle/to-save-settings.ts @@ -2,6 +2,13 @@ import type { IWorkflowSettings } from 'n8n-workflow'; import config from '@/config'; +export type ExecutionSaveSettings = { + error: boolean | 'all' | 'none'; + success: boolean | 'all' | 'none'; + manual: boolean; + progress: boolean; +}; + /** * Return whether a workflow execution is configured to be saved or not: * @@ -10,7 +17,7 @@ import config from '@/config'; * - `manual`: Whether to save successful or failed manual executions. * - `progress`: Whether to save execution progress, i.e. after each node's execution. */ -export function toSaveSettings(workflowSettings: IWorkflowSettings = {}) { +export function toSaveSettings(workflowSettings: IWorkflowSettings = {}): ExecutionSaveSettings { const DEFAULTS = { ERROR: config.getEnv('executions.saveDataOnError'), SUCCESS: config.getEnv('executions.saveDataOnSuccess'), diff --git a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts index 9cb681a7ef..e0ff5cb2e2 100644 --- a/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts +++ b/packages/cli/src/executions/__tests__/execution-recovery.service.test.ts @@ -3,6 +3,7 @@ import { stringify } from 'flatted'; import { mock } from 'jest-mock-extended'; import { InstanceSettings } from 'n8n-core'; import { randomInt } from 'n8n-workflow'; +import assert from 'node:assert'; import { ARTIFICIAL_TASK_DATA } from '@/constants'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; @@ -36,7 +37,6 @@ describe('ExecutionRecoveryService', () => { instanceSettings, push, executionRepository, - mock(), ); }); @@ -127,12 +127,15 @@ describe('ExecutionRecoveryService', () => { }); describe('if leader, with 1+ messages', () => { - test('should return `null` if execution succeeded', async () => { + test('for successful dataful execution, should return `null`', async () => { /** * Arrange */ const workflow = await createWorkflow(); - const execution = await createExecution({ status: 'success' }, workflow); + const execution = await createExecution( + { status: 'success', data: stringify({ runData: { foo: 'bar' } }) }, + workflow, + ); const messages = setupMessages(execution.id, 'Some workflow'); /** @@ -170,7 +173,38 @@ describe('ExecutionRecoveryService', () => { expect(amendedExecution).toBeNull(); }); - test('should update `status`, `stoppedAt` and `data` if last node did not finish', async () => { + test('for successful dataless execution, should update `status`, `stoppedAt` and `data`', async () => { + /** + * Arrange + */ + const workflow = await createWorkflow(); + const execution = await createExecution( + { + status: 'success', + data: stringify(undefined), // saved execution but likely crashed while saving high-volume data + }, + workflow, + ); + const messages = setupMessages(execution.id, 'Some workflow'); + + /** + * Act + */ + const amendedExecution = await executionRecoveryService.recoverFromLogs( + execution.id, + messages, + ); + + /** + * Assert + */ + assert(amendedExecution); + expect(amendedExecution.stoppedAt).not.toBe(execution.stoppedAt); + expect(amendedExecution.data).toEqual({ resultData: { runData: {} } }); + expect(amendedExecution.status).toBe('crashed'); + }); + + test('for running execution, should update `status`, `stoppedAt` and `data` if last node did not finish', async () => { /** * Arrange */ diff --git a/packages/cli/src/executions/execution-recovery.service.ts b/packages/cli/src/executions/execution-recovery.service.ts index 503e53d023..3a5d5a65b1 100644 --- a/packages/cli/src/executions/execution-recovery.service.ts +++ b/packages/cli/src/executions/execution-recovery.service.ts @@ -8,10 +8,9 @@ import { ARTIFICIAL_TASK_DATA } from '@/constants'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { NodeCrashedError } from '@/errors/node-crashed.error'; import { WorkflowCrashedError } from '@/errors/workflow-crashed.error'; -import { EventService } from '@/events/event.service'; +import { getLifecycleHooksForRegularMain } from '@/execution-lifecycle/execution-lifecycle-hooks'; import type { IExecutionResponse } from '@/interfaces'; import { Push } from '@/push'; -import { getWorkflowHooksMain } from '@/workflow-execute-additional-data'; // @TODO: Dependency cycle import type { EventMessageTypes } from '../eventbus/event-message-classes'; @@ -25,7 +24,6 @@ export class ExecutionRecoveryService { private readonly instanceSettings: InstanceSettings, private readonly push: Push, private readonly executionRepository: ExecutionRepository, - private readonly eventService: EventService, ) {} /** @@ -73,7 +71,7 @@ export class ExecutionRecoveryService { unflattenData: true, }); - if (!execution || execution.status === 'success') return null; + if (!execution || (execution.status === 'success' && execution.data)) return null; const runExecutionData = execution.data ?? { resultData: { runData: {} } }; @@ -176,20 +174,14 @@ export class ExecutionRecoveryService { private async runHooks(execution: IExecutionResponse) { execution.data ??= { resultData: { runData: {} } }; - this.eventService.emit('workflow-post-execute', { - workflow: execution.workflowData, - executionId: execution.id, - runData: execution, - }); - - const externalHooks = getWorkflowHooksMain( + const lifecycleHooks = getLifecycleHooksForRegularMain( { userId: '', workflowData: execution.workflowData, executionMode: execution.mode, executionData: execution.data, runData: execution.data.resultData.runData, - retryOf: execution.retryOf, + retryOf: execution.retryOf ?? undefined, }, execution.id, ); @@ -204,6 +196,6 @@ export class ExecutionRecoveryService { status: execution.status, }; - await externalHooks.executeHookFunctions('workflowExecuteAfter', [run]); + await lifecycleHooks.runHook('workflowExecuteAfter', [run]); } } diff --git a/packages/cli/src/external-hooks.ts b/packages/cli/src/external-hooks.ts index 8a0ba82c98..da8aa2d230 100644 --- a/packages/cli/src/external-hooks.ts +++ b/packages/cli/src/external-hooks.ts @@ -1,25 +1,104 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ +import type { FrontendSettings, UserUpdateRequestDto } from '@n8n/api-types'; +import type { ClientOAuth2Options } from '@n8n/client-oauth2'; +import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; +import { ErrorReporter, Logger } from 'n8n-core'; +import type { IRun, IWorkflowBase, Workflow, WorkflowExecuteMode } from 'n8n-workflow'; import { ApplicationError } from 'n8n-workflow'; +import type clientOAuth1 from 'oauth-1.0a'; -import config from '@/config'; +import type { AbstractServer } from '@/abstract-server'; +import type { Config } from '@/config'; +import type { TagEntity } from '@/databases/entities/tag-entity'; +import type { User } from '@/databases/entities/user'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import { SettingsRepository } from '@/databases/repositories/settings.repository'; import { UserRepository } from '@/databases/repositories/user.repository'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; -import type { IExternalHooksFileData, IExternalHooksFunctions } from '@/interfaces'; +import type { ICredentialsDb, PublicUser } from '@/interfaces'; + +type Repositories = { + User: UserRepository; + Settings: SettingsRepository; + Credentials: CredentialsRepository; + Workflow: WorkflowRepository; +}; + +type ExternalHooksMap = { + 'n8n.ready': [server: AbstractServer, config: Config]; + 'n8n.stop': never; + 'worker.ready': never; + + 'activeWorkflows.initialized': never; + + 'credentials.create': [encryptedData: ICredentialsDb]; + 'credentials.update': [newCredentialData: ICredentialsDb]; + 'credentials.delete': [credentialId: string]; + + 'frontend.settings': [frontendSettings: FrontendSettings]; + + 'mfa.beforeSetup': [user: User]; + + 'oauth1.authenticate': [ + oAuthOptions: clientOAuth1.Options, + oauthRequestData: { oauth_callback: string }, + ]; + 'oauth2.authenticate': [oAuthOptions: ClientOAuth2Options]; + 'oauth2.callback': [oAuthOptions: ClientOAuth2Options]; + + 'tag.beforeCreate': [tag: TagEntity]; + 'tag.afterCreate': [tag: TagEntity]; + 'tag.beforeUpdate': [tag: TagEntity]; + 'tag.afterUpdate': [tag: TagEntity]; + 'tag.beforeDelete': [tagId: string]; + 'tag.afterDelete': [tagId: string]; + + 'user.deleted': [user: PublicUser]; + 'user.profile.beforeUpdate': [ + userId: string, + currentEmail: string, + payload: UserUpdateRequestDto, + ]; + 'user.profile.update': [currentEmail: string, publicUser: PublicUser]; + 'user.password.update': [updatedEmail: string, updatedPassword: string]; + 'user.invited': [emails: string[]]; + + 'workflow.create': [createdWorkflow: IWorkflowBase]; + 'workflow.afterCreate': [createdWorkflow: IWorkflowBase]; + 'workflow.activate': [updatedWorkflow: IWorkflowBase]; + 'workflow.update': [updatedWorkflow: IWorkflowBase]; + 'workflow.afterUpdate': [updatedWorkflow: IWorkflowBase]; + 'workflow.delete': [workflowId: string]; + 'workflow.afterDelete': [workflowId: string]; + + 'workflow.preExecute': [workflow: Workflow, mode: WorkflowExecuteMode]; + 'workflow.postExecute': [ + fullRunData: IRun | undefined, + workflowData: IWorkflowBase, + executionId: string, + ]; +}; +type HookNames = keyof ExternalHooksMap; + +// TODO: Derive this type from Hooks +interface IExternalHooksFileData { + [Resource: string]: { + [Operation: string]: Array<(...args: unknown[]) => Promise>; + }; +} @Service() export class ExternalHooks { - externalHooks: { - [key: string]: Array<() => {}>; + private readonly registered: { + [hookName in HookNames]?: Array<(...args: ExternalHooksMap[hookName]) => Promise>; } = {}; - private initDidRun = false; - - private dbCollections: IExternalHooksFunctions['dbCollections']; + private readonly dbCollections: Repositories; constructor( + private readonly logger: Logger, + private readonly errorReporter: ErrorReporter, + private readonly globalConfig: GlobalConfig, userRepository: UserRepository, settingsRepository: SettingsRepository, credentialsRepository: CredentialsRepository, @@ -33,72 +112,58 @@ export class ExternalHooks { }; } - async init(): Promise { - if (this.initDidRun) { - return; - } - - await this.loadHooksFiles(); - - this.initDidRun = true; - } - - private async loadHooksFiles() { - const externalHookFiles = config.getEnv('externalHookFiles').split(':'); + async init() { + const externalHookFiles = this.globalConfig.externalHooks.files; // Load all the provided hook-files for (let hookFilePath of externalHookFiles) { hookFilePath = hookFilePath.trim(); - if (hookFilePath !== '') { - try { - const hookFile = require(hookFilePath) as IExternalHooksFileData; - this.loadHooks(hookFile); - } catch (e) { - const error = e instanceof Error ? e : new Error(`${e}`); + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const hookFile = require(hookFilePath) as IExternalHooksFileData; + this.loadHooks(hookFile); + } catch (e) { + const error = e instanceof Error ? e : new Error(`${e}`); - throw new ApplicationError('Problem loading external hook file', { - extra: { errorMessage: error.message, hookFilePath }, - cause: error, - }); - } + throw new ApplicationError('Problem loading external hook file', { + extra: { errorMessage: error.message, hookFilePath }, + cause: error, + }); } } } private loadHooks(hookFileData: IExternalHooksFileData) { - for (const resource of Object.keys(hookFileData)) { - for (const operation of Object.keys(hookFileData[resource])) { - // Save all the hook functions directly under their string - // format in an array - const hookString = `${resource}.${operation}`; - if (this.externalHooks[hookString] === undefined) { - this.externalHooks[hookString] = []; - } - - // eslint-disable-next-line prefer-spread - this.externalHooks[hookString].push.apply( - this.externalHooks[hookString], - hookFileData[resource][operation], - ); + const { registered } = this; + for (const [resource, operations] of Object.entries(hookFileData)) { + for (const operation of Object.keys(operations)) { + const hookName = `${resource}.${operation}` as HookNames; + registered[hookName] ??= []; + registered[hookName].push(...operations[operation]); } } } - async run(hookName: string, hookParameters?: any[]): Promise { - if (this.externalHooks[hookName] === undefined) { - return; + async run( + hookName: HookName, + hookParameters?: ExternalHooksMap[HookName], + ): Promise { + const { registered, dbCollections } = this; + const hookFunctions = registered[hookName]; + if (!hookFunctions?.length) return; + + const context = { dbCollections }; + + for (const hookFunction of hookFunctions) { + try { + await hookFunction.apply(context, hookParameters); + } catch (cause) { + this.logger.error(`There was a problem running hook "${hookName}"`); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const error = new ApplicationError(`External hook "${hookName}" failed`, { cause }); + this.errorReporter.error(error, { level: 'fatal' }); + throw error; + } } - - const externalHookFunctions: IExternalHooksFunctions = { - dbCollections: this.dbCollections, - }; - - for (const externalHookFunction of this.externalHooks[hookName]) { - await externalHookFunction.apply(externalHookFunctions, hookParameters); - } - } - - exists(hookName: string): boolean { - return !!this.externalHooks[hookName]; } } diff --git a/packages/cli/src/interfaces.ts b/packages/cli/src/interfaces.ts index e50b96c384..c5cf76bd32 100644 --- a/packages/cli/src/interfaces.ts +++ b/packages/cli/src/interfaces.ts @@ -31,10 +31,6 @@ import type { AuthProviderType } from '@/databases/entities/auth-identity'; import type { SharedCredentials } from '@/databases/entities/shared-credentials'; import type { TagEntity } from '@/databases/entities/tag-entity'; import type { AssignableRole, GlobalRole, User } from '@/databases/entities/user'; -import type { CredentialsRepository } from '@/databases/repositories/credentials.repository'; -import type { SettingsRepository } from '@/databases/repositories/settings.repository'; -import type { UserRepository } from '@/databases/repositories/user.repository'; -import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import type { LICENSE_FEATURES, LICENSE_QUOTAS } from './constants'; import type { ExternalHooks } from './external-hooks'; @@ -220,46 +216,6 @@ export interface IExecutingWorkflowData { status: ExecutionStatus; } -export interface IExternalHooks { - credentials?: { - create?: Array<{ - (this: IExternalHooksFunctions, credentialsData: ICredentialsEncrypted): Promise; - }>; - delete?: Array<{ (this: IExternalHooksFunctions, credentialId: string): Promise }>; - update?: Array<{ - (this: IExternalHooksFunctions, credentialsData: ICredentialsDb): Promise; - }>; - }; - workflow?: { - activate?: Array<{ (this: IExternalHooksFunctions, workflowData: IWorkflowDb): Promise }>; - create?: Array<{ (this: IExternalHooksFunctions, workflowData: IWorkflowBase): Promise }>; - delete?: Array<{ (this: IExternalHooksFunctions, workflowId: string): Promise }>; - execute?: Array<{ - ( - this: IExternalHooksFunctions, - workflowData: IWorkflowDb, - mode: WorkflowExecuteMode, - ): Promise; - }>; - update?: Array<{ (this: IExternalHooksFunctions, workflowData: IWorkflowDb): Promise }>; - }; -} - -export interface IExternalHooksFileData { - [key: string]: { - [key: string]: Array<(...args: any[]) => Promise>; - }; -} - -export interface IExternalHooksFunctions { - dbCollections: { - User: UserRepository; - Settings: SettingsRepository; - Credentials: CredentialsRepository; - Workflow: WorkflowRepository; - }; -} - export interface IPersonalizationSurveyAnswers { email: string | null; codingSkill: string | null; diff --git a/packages/cli/src/license.ts b/packages/cli/src/license.ts index 7a747530c2..c0bece9927 100644 --- a/packages/cli/src/license.ts +++ b/packages/cli/src/license.ts @@ -18,6 +18,9 @@ import { } from './constants'; import type { BooleanLicenseFeature, NumericLicenseFeature } from './interfaces'; +const LICENSE_RENEWAL_DISABLED_WARNING = + 'Automatic license renewal is disabled. The license will not renew automatically, and access to licensed features may be lost!'; + export type FeatureReturnType = Partial< { planName: string; @@ -40,26 +43,10 @@ export class License { this.logger = this.logger.scoped('license'); } - /** - * Whether this instance should renew the license - on init and periodically. - */ - private renewalEnabled() { - if (this.instanceSettings.instanceType !== 'main') return false; - const autoRenewEnabled = this.globalConfig.license.autoRenewalEnabled; - - /** - * In multi-main setup, all mains start off with `unset` status and so renewal disabled. - * On becoming leader or follower, each will enable or disable renewal, respectively. - * This ensures the mains do not cause a 429 (too many requests) on license init. - */ - if (this.globalConfig.multiMainSetup.enabled) { - return autoRenewEnabled && this.instanceSettings.isLeader; - } - - return autoRenewEnabled; - } - - async init(forceRecreate = false) { + async init({ + forceRecreate = false, + isCli = false, + }: { forceRecreate?: boolean; isCli?: boolean } = {}) { if (this.manager && !forceRecreate) { this.logger.warn('License manager already initialized or shutting down'); return; @@ -87,15 +74,23 @@ export class License { ? async () => await this.licenseMetricsService.collectPassthroughData() : async () => ({}); - const renewalEnabled = this.renewalEnabled(); + const { isLeader } = this.instanceSettings; + const { autoRenewalEnabled } = this.globalConfig.license; + const eligibleToRenew = isCli || isLeader; + + const shouldRenew = eligibleToRenew && autoRenewalEnabled; + + if (eligibleToRenew && !autoRenewalEnabled) { + this.logger.warn(LICENSE_RENEWAL_DISABLED_WARNING); + } try { this.manager = new LicenseManager({ server, tenantId: this.globalConfig.license.tenantId, productIdentifier: `n8n-${N8N_VERSION}`, - autoRenewEnabled: renewalEnabled, - renewOnInit: renewalEnabled, + autoRenewEnabled: shouldRenew, + renewOnInit: shouldRenew, autoRenewOffset, offlineMode, logger: this.logger, @@ -275,7 +270,7 @@ export class License { return this.isFeatureEnabled(LICENSE_FEATURES.BINARY_DATA_S3); } - isMultipleMainInstancesLicensed() { + isMultiMainLicensed() { return this.isFeatureEnabled(LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES); } @@ -335,7 +330,7 @@ export class License { } /** - * Helper function to get the main plan for a license + * Helper function to get the latest main plan for a license */ getMainPlan(): TEntitlement | undefined { if (!this.manager) { @@ -347,6 +342,8 @@ export class License { return undefined; } + entitlements.sort((a, b) => b.validFrom.getTime() - a.validFrom.getTime()); + return entitlements.find( (entitlement) => (entitlement.productMetadata?.terms as { isMainPlan?: boolean })?.isMainPlan, ); @@ -361,6 +358,10 @@ export class License { return this.getFeatureValue(LICENSE_QUOTAS.USERS_LIMIT) ?? UNLIMITED_LICENSE_QUOTA; } + getApiKeysPerUserLimit() { + return this.getFeatureValue(LICENSE_QUOTAS.API_KEYS_PER_USER_LIMIT) ?? 1; + } + getTriggerLimit() { return this.getFeatureValue(LICENSE_QUOTAS.TRIGGER_LIMIT) ?? UNLIMITED_LICENSE_QUOTA; } @@ -400,8 +401,10 @@ export class License { } async reinit() { - this.manager?.reset(); - await this.init(true); + if (this.manager) { + await this.manager.reset(); + } + await this.init({ forceRecreate: true }); this.logger.debug('License reinitialized'); } } diff --git a/packages/cli/src/load-nodes-and-credentials.ts b/packages/cli/src/load-nodes-and-credentials.ts index b027192388..dadf44c7ab 100644 --- a/packages/cli/src/load-nodes-and-credentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -174,6 +174,29 @@ export class LoadNodesAndCredentials { return isContainedWithin(loader.directory, filePath) ? filePath : undefined; } + resolveSchema({ + node, + version, + resource, + operation, + }: { + node: string; + version: string; + resource?: string; + operation?: string; + }): string | undefined { + const nodePath = this.known.nodes[node]?.sourcePath; + if (!nodePath) { + return undefined; + } + + const nodeParentPath = path.dirname(nodePath); + const schemaPath = ['__schema__', `v${version}`, resource, operation].filter(Boolean).join('/'); + const filePath = path.resolve(nodeParentPath, schemaPath + '.json'); + + return isContainedWithin(nodeParentPath, filePath) ? filePath : undefined; + } + getCustomDirectories(): string[] { const customDirectories = [this.instanceSettings.customExtensionDir]; @@ -323,7 +346,15 @@ export class LoadNodesAndCredentials { name: `${packageName}.${name}`, })), ); - this.types.credentials = this.types.credentials.concat(types.credentials); + this.types.credentials = this.types.credentials.concat( + types.credentials.map(({ supportedNodes, ...rest }) => ({ + ...rest, + supportedNodes: + loader instanceof PackageDirectoryLoader + ? supportedNodes?.map((nodeName) => `${loader.packageName}.${nodeName}`) + : undefined, + })), + ); // Nodes and credentials that have been loaded immediately for (const nodeTypeName in loader.nodeTypes) { @@ -453,14 +484,6 @@ export class LoadNodesAndCredentials { placeholder: `e.g. ${item.description.description}`, }; - const noticeProp: INodeProperties = { - displayName: - "Use the expression {{ $fromAI('placeholder_name') }} for any data to be filled by the model", - name: 'notice', - type: 'notice', - default: '', - }; - item.description.properties.unshift(descProp); // If node has resource or operation we can determine pre-populate tool description based on it @@ -474,8 +497,6 @@ export class LoadNodesAndCredentials { }, }; } - - item.description.properties.unshift(noticeProp); } } diff --git a/packages/cli/src/manual-execution.service.ts b/packages/cli/src/manual-execution.service.ts index fd6c27215f..4d6754ff4e 100644 --- a/packages/cli/src/manual-execution.service.ts +++ b/packages/cli/src/manual-execution.service.ts @@ -23,6 +23,13 @@ export class ManualExecutionService { getExecutionStartNode(data: IWorkflowExecutionDataProcess, workflow: Workflow) { let startNode; + + // If the user chose a trigger to start from we honor this. + if (data.triggerToStartFrom?.name) { + startNode = workflow.getNode(data.triggerToStartFrom.name) ?? undefined; + } + + // Old logic for partial executions v1 if ( data.startNodes?.length === 1 && Object.keys(data.pinData ?? {}).includes(data.startNodes[0].name) @@ -71,7 +78,11 @@ export class ManualExecutionService { }, }; - const workflowExecute = new WorkflowExecute(additionalData, 'manual', executionData); + const workflowExecute = new WorkflowExecute( + additionalData, + data.executionMode, + executionData, + ); return workflowExecute.processRunExecutionData(workflow); } else if ( data.runData === undefined || @@ -101,7 +112,7 @@ export class ManualExecutionService { // Execute only the nodes between start and destination nodes const workflowExecute = new WorkflowExecute(additionalData, data.executionMode); - if (data.partialExecutionVersion === '1') { + if (data.partialExecutionVersion === 2) { return workflowExecute.runPartialWorkflow2( workflow, data.runData, diff --git a/packages/cli/src/metrics/license-metrics.service.ts b/packages/cli/src/metrics/license-metrics.service.ts index bd4ca6055b..1531809cd2 100644 --- a/packages/cli/src/metrics/license-metrics.service.ts +++ b/packages/cli/src/metrics/license-metrics.service.ts @@ -37,7 +37,9 @@ export class LicenseMetricsService { async collectPassthroughData() { return { - activeWorkflowIds: await this.workflowRepository.getActiveIds(), + // Get only the first 1000 active workflow IDs to avoid sending too much data to License Server + // Passthrough data is forwarded to Telemetry for further analysis, such as quota excesses + activeWorkflowIds: await this.workflowRepository.getActiveIds({ maxResults: 1000 }), }; } } diff --git a/packages/cli/src/middlewares/list-query/__tests__/list-query.test.ts b/packages/cli/src/middlewares/list-query/__tests__/list-query.test.ts index 0e77792a43..3ced89ed49 100644 --- a/packages/cli/src/middlewares/list-query/__tests__/list-query.test.ts +++ b/packages/cli/src/middlewares/list-query/__tests__/list-query.test.ts @@ -6,6 +6,8 @@ import { selectListQueryMiddleware } from '@/middlewares/list-query/select'; import type { ListQuery } from '@/requests'; import * as ResponseHelper from '@/response-helper'; +import { sortByQueryMiddleware } from '../sort-by'; + describe('List query middleware', () => { let mockReq: ListQuery.Request; let mockRes: Response; @@ -174,6 +176,84 @@ describe('List query middleware', () => { }); }); + describe('Query sort by', () => { + const validCases: Array<{ name: string; value: ListQuery.Workflow.SortOrder }> = [ + { + name: 'sorting by name asc', + value: 'name:asc', + }, + { + name: 'sorting by name desc', + value: 'name:desc', + }, + { + name: 'sorting by createdAt asc', + value: 'createdAt:asc', + }, + { + name: 'sorting by createdAt desc', + value: 'createdAt:desc', + }, + { + name: 'sorting by updatedAt asc', + value: 'updatedAt:asc', + }, + { + name: 'sorting by updatedAt desc', + value: 'updatedAt:desc', + }, + ]; + + const invalidCases: Array<{ name: string; value: string }> = [ + { + name: 'sorting by invalid column', + value: 'test:asc', + }, + { + name: 'sorting by valid column without order', + value: 'name', + }, + { + name: 'sorting by valid column with invalid order', + value: 'name:test', + }, + ]; + + test.each(validCases)('should succeed validation when $name', async ({ value }) => { + mockReq.query = { + sortBy: value, + }; + + sortByQueryMiddleware(...args); + + expect(mockReq.listQueryOptions).toMatchObject( + expect.objectContaining({ + sortBy: value, + }), + ); + expect(nextFn).toBeCalledTimes(1); + }); + + test.each(invalidCases)('should fail validation when $name', async ({ value }) => { + mockReq.query = { + sortBy: value as ListQuery.Workflow.SortOrder, + }; + + sortByQueryMiddleware(...args); + + expect(sendErrorResponse).toHaveBeenCalledTimes(1); + }); + + test('should not pass sortBy to listQueryOptions if not provided', async () => { + mockReq.query = {}; + + sortByQueryMiddleware(...args); + + expect(mockReq.listQueryOptions).toBeUndefined(); + expect(nextFn).toBeCalledTimes(1); + }); + }); + describe('Combinations', () => { test('should combine filter with select', async () => { mockReq.query = { filter: '{ "name": "My Workflow" }', select: '["name", "id"]' }; diff --git a/packages/cli/src/middlewares/list-query/dtos/workflow.sort-by.dto.ts b/packages/cli/src/middlewares/list-query/dtos/workflow.sort-by.dto.ts new file mode 100644 index 0000000000..80a0ca3a74 --- /dev/null +++ b/packages/cli/src/middlewares/list-query/dtos/workflow.sort-by.dto.ts @@ -0,0 +1,22 @@ +import type { ValidatorConstraintInterface, ValidationArguments } from 'class-validator'; +import { IsString, Validate, ValidatorConstraint } from 'class-validator'; + +@ValidatorConstraint({ name: 'WorkflowSortByParameter', async: false }) +export class WorkflowSortByParameter implements ValidatorConstraintInterface { + validate(text: string, _: ValidationArguments) { + const [column, order] = text.split(':'); + if (!column || !order) return false; + + return ['name', 'createdAt', 'updatedAt'].includes(column) && ['asc', 'desc'].includes(order); + } + + defaultMessage(_: ValidationArguments) { + return 'Invalid value for sortBy parameter'; + } +} + +export class WorkflowSorting { + @IsString() + @Validate(WorkflowSortByParameter) + sortBy?: string; +} diff --git a/packages/cli/src/middlewares/list-query/index.ts b/packages/cli/src/middlewares/list-query/index.ts index a766b43bd0..2e4781337a 100644 --- a/packages/cli/src/middlewares/list-query/index.ts +++ b/packages/cli/src/middlewares/list-query/index.ts @@ -1,10 +1,11 @@ -import type { NextFunction, Response } from 'express'; +import { type NextFunction, type Response } from 'express'; import type { ListQuery } from '@/requests'; import { filterListQueryMiddleware } from './filter'; import { paginationListQueryMiddleware } from './pagination'; import { selectListQueryMiddleware } from './select'; +import { sortByQueryMiddleware } from './sort-by'; export type ListQueryMiddleware = ( req: ListQuery.Request, @@ -16,4 +17,5 @@ export const listQueryMiddleware: ListQueryMiddleware[] = [ filterListQueryMiddleware, selectListQueryMiddleware, paginationListQueryMiddleware, + sortByQueryMiddleware, ]; diff --git a/packages/cli/src/middlewares/list-query/sort-by.ts b/packages/cli/src/middlewares/list-query/sort-by.ts new file mode 100644 index 0000000000..01aca505af --- /dev/null +++ b/packages/cli/src/middlewares/list-query/sort-by.ts @@ -0,0 +1,39 @@ +import { plainToInstance } from 'class-transformer'; +import { validateSync } from 'class-validator'; +import type { RequestHandler } from 'express'; +import { ApplicationError } from 'n8n-workflow'; + +import type { ListQuery } from '@/requests'; +import * as ResponseHelper from '@/response-helper'; +import { toError } from '@/utils'; + +import { WorkflowSorting } from './dtos/workflow.sort-by.dto'; + +export const sortByQueryMiddleware: RequestHandler = (req: ListQuery.Request, res, next) => { + const { sortBy } = req.query; + + if (!sortBy) return next(); + + let SortBy; + + try { + if (req.baseUrl.endsWith('workflows')) { + SortBy = WorkflowSorting; + } else { + return next(); + } + + const validationResponse = validateSync(plainToInstance(SortBy, { sortBy })); + + if (validationResponse.length) { + const validationError = validationResponse[0]; + throw new ApplicationError(validationError.constraints?.workflowSortBy ?? ''); + } + + req.listQueryOptions = { ...req.listQueryOptions, sortBy }; + + next(); + } catch (maybeError) { + ResponseHelper.sendErrorResponse(res, toError(maybeError)); + } +}; diff --git a/packages/cli/src/node-types.ts b/packages/cli/src/node-types.ts index 77d872dc26..bb0a5fbbc7 100644 --- a/packages/cli/src/node-types.ts +++ b/packages/cli/src/node-types.ts @@ -1,7 +1,10 @@ +import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; import type { NeededNodeType } from '@n8n/task-runner'; import type { Dirent } from 'fs'; import { readdir } from 'fs/promises'; +import { RoutingNode } from 'n8n-core'; +import type { ExecuteContext } from 'n8n-core'; import type { INodeType, INodeTypeDescription, INodeTypes, IVersionedNodeType } from 'n8n-workflow'; import { ApplicationError, NodeHelpers } from 'n8n-workflow'; import { join, dirname } from 'path'; @@ -10,7 +13,10 @@ import { LoadNodesAndCredentials } from './load-nodes-and-credentials'; @Service() export class NodeTypes implements INodeTypes { - constructor(private loadNodesAndCredentials: LoadNodesAndCredentials) {} + constructor( + private readonly globalConfig: GlobalConfig, + private readonly loadNodesAndCredentials: LoadNodesAndCredentials, + ) {} /** * Variant of `getByNameAndVersion` that includes the node's source path, used to locate a node's translations. @@ -31,14 +37,38 @@ export class NodeTypes implements INodeTypes { getByNameAndVersion(nodeType: string, version?: number): INodeType { const origType = nodeType; - const toolRequested = nodeType.startsWith('n8n-nodes-base') && nodeType.endsWith('Tool'); + + const { communityPackages } = this.globalConfig.nodes; + const allowToolUsage = communityPackages.allowToolUsage + ? true + : nodeType.startsWith('n8n-nodes-base'); + const toolRequested = nodeType.endsWith('Tool'); + // Make sure the nodeType to actually get from disk is the un-wrapped type - if (toolRequested) { + if (allowToolUsage && toolRequested) { nodeType = nodeType.replace(/Tool$/, ''); } const node = this.loadNodesAndCredentials.getNode(nodeType); const versionedNodeType = NodeHelpers.getVersionedNodeType(node.type, version); + if (toolRequested && typeof versionedNodeType.supplyData === 'function') { + throw new ApplicationError('Node already has a `supplyData` method', { extra: { nodeType } }); + } + + if ( + !versionedNodeType.execute && + !versionedNodeType.poll && + !versionedNodeType.trigger && + !versionedNodeType.webhook && + !versionedNodeType.methods + ) { + versionedNodeType.execute = async function (this: ExecuteContext) { + const routingNode = new RoutingNode(this, versionedNodeType); + const data = await routingNode.runNode(); + return data ?? []; + }; + } + if (!toolRequested) return versionedNodeType; if (!versionedNodeType.description.usableAsTool) diff --git a/packages/cli/src/public-api/types.ts b/packages/cli/src/public-api/types.ts index b10d2f81bd..14e586dfdb 100644 --- a/packages/cli/src/public-api/types.ts +++ b/packages/cli/src/public-api/types.ts @@ -173,16 +173,6 @@ export interface IJsonSchema { required: string[]; } -export class SourceControlPull { - force?: boolean; - - variables?: { [key: string]: string }; -} - -export declare namespace PublicSourceControlRequest { - type Pull = AuthenticatedRequest<{}, {}, SourceControlPull, {}>; -} - // ---------------------------------- // /audit // ---------------------------------- diff --git a/packages/cli/src/public-api/v1/handlers/projects/projects.handler.ts b/packages/cli/src/public-api/v1/handlers/projects/projects.handler.ts index a693058f93..c55b10053d 100644 --- a/packages/cli/src/public-api/v1/handlers/projects/projects.handler.ts +++ b/packages/cli/src/public-api/v1/handlers/projects/projects.handler.ts @@ -1,25 +1,28 @@ +import { CreateProjectDto, DeleteProjectDto, UpdateProjectDto } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { Response } from 'express'; import { ProjectController } from '@/controllers/project.controller'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import type { PaginatedRequest } from '@/public-api/types'; -import type { ProjectRequest } from '@/requests'; +import type { AuthenticatedRequest } from '@/requests'; import { globalScope, isLicensed, validCursor } from '../../shared/middlewares/global.middleware'; import { encodeNextCursor } from '../../shared/services/pagination.service'; -type Create = ProjectRequest.Create; -type Update = ProjectRequest.Update; -type Delete = ProjectRequest.Delete; type GetAll = PaginatedRequest; export = { createProject: [ isLicensed('feat:projectRole:admin'), globalScope('project:create'), - async (req: Create, res: Response) => { - const project = await Container.get(ProjectController).createProject(req); + async (req: AuthenticatedRequest, res: Response) => { + const payload = CreateProjectDto.safeParse(req.body); + if (payload.error) { + return res.status(400).json(payload.error.errors[0]); + } + + const project = await Container.get(ProjectController).createProject(req, res, payload.data); return res.status(201).json(project); }, @@ -27,8 +30,18 @@ export = { updateProject: [ isLicensed('feat:projectRole:admin'), globalScope('project:update'), - async (req: Update, res: Response) => { - await Container.get(ProjectController).updateProject(req); + async (req: AuthenticatedRequest<{ projectId: string }>, res: Response) => { + const payload = UpdateProjectDto.safeParse(req.body); + if (payload.error) { + return res.status(400).json(payload.error.errors[0]); + } + + await Container.get(ProjectController).updateProject( + req, + res, + payload.data, + req.params.projectId, + ); return res.status(204).send(); }, @@ -36,8 +49,18 @@ export = { deleteProject: [ isLicensed('feat:projectRole:admin'), globalScope('project:delete'), - async (req: Delete, res: Response) => { - await Container.get(ProjectController).deleteProject(req); + async (req: AuthenticatedRequest<{ projectId: string }>, res: Response) => { + const query = DeleteProjectDto.safeParse(req.query); + if (query.error) { + return res.status(400).json(query.error.errors[0]); + } + + await Container.get(ProjectController).deleteProject( + req, + res, + query.data, + req.params.projectId, + ); return res.status(204).send(); }, diff --git a/packages/cli/src/public-api/v1/handlers/source-control/source-control.handler.ts b/packages/cli/src/public-api/v1/handlers/source-control/source-control.handler.ts index 646a3f075e..4a0729ef6e 100644 --- a/packages/cli/src/public-api/v1/handlers/source-control/source-control.handler.ts +++ b/packages/cli/src/public-api/v1/handlers/source-control/source-control.handler.ts @@ -1,3 +1,4 @@ +import { PullWorkFolderRequestDto } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type express from 'express'; import type { StatusResult } from 'simple-git'; @@ -10,15 +11,15 @@ import { SourceControlPreferencesService } from '@/environments.ee/source-contro import { SourceControlService } from '@/environments.ee/source-control/source-control.service.ee'; import type { ImportResult } from '@/environments.ee/source-control/types/import-result'; import { EventService } from '@/events/event.service'; +import type { AuthenticatedRequest } from '@/requests'; -import type { PublicSourceControlRequest } from '../../../types'; import { globalScope } from '../../shared/middlewares/global.middleware'; export = { pull: [ globalScope('sourceControl:pull'), async ( - req: PublicSourceControlRequest.Pull, + req: AuthenticatedRequest, res: express.Response, ): Promise> => { const sourceControlPreferencesService = Container.get(SourceControlPreferencesService); @@ -33,17 +34,14 @@ export = { .json({ status: 'Error', message: 'Source Control is not connected to a repository' }); } try { + const payload = PullWorkFolderRequestDto.parse(req.body); const sourceControlService = Container.get(SourceControlService); - const result = await sourceControlService.pullWorkfolder({ - force: req.body.force, - variables: req.body.variables, - userId: req.user.id, - }); + const result = await sourceControlService.pullWorkfolder(req.user, payload); if (result.statusCode === 200) { Container.get(EventService).emit('source-control-user-pulled-api', { ...getTrackingInformationFromPullResult(result.statusResult), - forced: req.body.force ?? false, + forced: payload.force ?? false, }); return res.status(200).send(result.statusResult); } else { diff --git a/packages/cli/src/public-api/v1/handlers/workflows/workflows.handler.ts b/packages/cli/src/public-api/v1/handlers/workflows/workflows.handler.ts index b79ea4547b..77a07fea0f 100644 --- a/packages/cli/src/public-api/v1/handlers/workflows/workflows.handler.ts +++ b/packages/cli/src/public-api/v1/handlers/workflows/workflows.handler.ts @@ -1,14 +1,14 @@ +import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import -import type { FindOptionsWhere } from '@n8n/typeorm'; -// eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In, Like, QueryFailedError } from '@n8n/typeorm'; +// eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import +import type { FindOptionsWhere } from '@n8n/typeorm'; import type express from 'express'; import { v4 as uuid } from 'uuid'; import { z } from 'zod'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; -import config from '@/config'; import { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository'; @@ -111,7 +111,7 @@ export = { id, req.user, ['workflow:read'], - { includeTags: !config.getEnv('workflowTagsDisabled') }, + { includeTags: !Container.get(GlobalConfig).tags.disabled }, ); if (!workflow) { @@ -209,7 +209,7 @@ export = { skip: offset, take: limit, where, - ...(!config.getEnv('workflowTagsDisabled') && { relations: ['tags'] }), + ...(!Container.get(GlobalConfig).tags.disabled && { relations: ['tags'] }), }); if (excludePinnedData) { @@ -379,7 +379,7 @@ export = { async (req: WorkflowRequest.GetTags, res: express.Response): Promise => { const { id } = req.params; - if (config.getEnv('workflowTagsDisabled')) { + if (Container.get(GlobalConfig).tags.disabled) { return res.status(400).json({ message: 'Workflow Tags Disabled' }); } @@ -406,7 +406,7 @@ export = { const { id } = req.params; const newTags = req.body.map((newTag) => newTag.id); - if (config.getEnv('workflowTagsDisabled')) { + if (Container.get(GlobalConfig).tags.disabled) { return res.status(400).json({ message: 'Workflow Tags Disabled' }); } diff --git a/packages/cli/src/public-api/v1/handlers/workflows/workflows.service.ts b/packages/cli/src/public-api/v1/handlers/workflows/workflows.service.ts index 53e67d8b0d..131919d096 100644 --- a/packages/cli/src/public-api/v1/handlers/workflows/workflows.service.ts +++ b/packages/cli/src/public-api/v1/handlers/workflows/workflows.service.ts @@ -1,7 +1,7 @@ +import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; -import config from '@/config'; import type { Project } from '@/databases/entities/project'; import { SharedWorkflow, type WorkflowSharingRole } from '@/databases/entities/shared-workflow'; import type { User } from '@/databases/entities/user'; @@ -46,7 +46,10 @@ export async function getSharedWorkflow( ...(!['global:owner', 'global:admin'].includes(user.role) && { userId: user.id }), ...(workflowId && { workflowId }), }, - relations: [...insertIf(!config.getEnv('workflowTagsDisabled'), ['workflow.tags']), 'workflow'], + relations: [ + ...insertIf(!Container.get(GlobalConfig).tags.disabled, ['workflow.tags']), + 'workflow', + ], }); } diff --git a/packages/cli/src/push/index.ts b/packages/cli/src/push/index.ts index e795df5722..4c2d8ac032 100644 --- a/packages/cli/src/push/index.ts +++ b/packages/cli/src/push/index.ts @@ -169,8 +169,12 @@ export class Push extends TypedEmitter { this.logger.warn(`Size of "${type}" (${eventMb} MB) exceeds max size ${maxMb} MB. Trimming...`); - if (type === 'nodeExecuteAfter') pushMsgCopy.data.data.data = TRIMMED_TASK_DATA_CONNECTIONS; - else if (type === 'executionFinished') pushMsgCopy.data.rawData = ''; // prompt client to fetch from DB + if (type === 'nodeExecuteAfter') { + pushMsgCopy.data.itemCount = pushMsgCopy.data.data.data?.main[0]?.length ?? 1; + pushMsgCopy.data.data.data = TRIMMED_TASK_DATA_CONNECTIONS; + } else if (type === 'executionFinished') { + pushMsgCopy.data.rawData = ''; // prompt client to fetch from DB + } void this.publisher.publishCommand({ command: 'relay-execution-lifecycle-event', diff --git a/packages/cli/src/requests.ts b/packages/cli/src/requests.ts index 5776549566..f57b8b858f 100644 --- a/packages/cli/src/requests.ts +++ b/packages/cli/src/requests.ts @@ -1,3 +1,4 @@ +import type { ProjectIcon, ProjectRole, ProjectType } from '@n8n/api-types'; import type { Scope } from '@n8n/permissions'; import type express from 'express'; import type { @@ -9,14 +10,13 @@ import type { } from 'n8n-workflow'; import type { CredentialsEntity } from '@/databases/entities/credentials-entity'; -import type { Project, ProjectIcon, ProjectType } from '@/databases/entities/project'; +import type { Project } from '@/databases/entities/project'; import type { AssignableRole, GlobalRole, User } from '@/databases/entities/user'; import type { Variables } from '@/databases/entities/variables'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import type { WorkflowHistory } from '@/databases/entities/workflow-history'; import type { SecretsProvider, SecretsProviderState } from '@/interfaces'; -import type { ProjectRole } from './databases/entities/project-relation'; import type { ScopesField } from './services/role.service'; export type APIRequest< @@ -62,6 +62,7 @@ export namespace ListQuery { skip?: string; take?: string; select?: string; + sortBy?: string; }; export type Options = { @@ -69,6 +70,7 @@ export namespace ListQuery { select?: Record; skip?: number; take?: number; + sortBy?: string; }; /** @@ -82,6 +84,10 @@ export namespace ListQuery { type SharedField = Partial>; + type SortingField = 'createdAt' | 'updatedAt' | 'name'; + + export type SortOrder = `${SortingField}:asc` | `${SortingField}:desc`; + type OwnedByField = { ownedBy: SlimUser | null; homeProject: SlimProject | null }; export type Plain = BaseFields; @@ -175,14 +181,6 @@ export declare namespace CredentialRequest { >; } -// ---------------------------------- -// /api-keys -// ---------------------------------- - -export declare namespace ApiKeysRequest { - export type DeleteAPIKey = AuthenticatedRequest<{ id: string }>; -} - // ---------------------------------- // /me // ---------------------------------- @@ -264,17 +262,6 @@ export declare namespace OAuthRequest { } } -// ---------------------------------- -// /tags -// ---------------------------------- - -export declare namespace TagsRequest { - type GetAll = AuthenticatedRequest<{}, {}, {}, { withUsageCount: string }>; - type Create = AuthenticatedRequest<{}, {}, { name: string }>; - type Update = AuthenticatedRequest<{ id: string }, {}, { name: string }>; - type Delete = AuthenticatedRequest<{ id: string }>; -} - // ---------------------------------- // /annotation-tags // ---------------------------------- @@ -388,32 +375,10 @@ export declare namespace ActiveWorkflowRequest { // ---------------------------------- export declare namespace ProjectRequest { - type GetAll = AuthenticatedRequest<{}, Project[]>; - - type Create = AuthenticatedRequest< - {}, - Project, - { - name: string; - icon?: ProjectIcon; - } - >; - - type GetMyProjects = AuthenticatedRequest< - {}, - Array, - {}, - { - includeScopes?: boolean; - } - >; type GetMyProjectsResponse = Array< Project & { role: ProjectRole | GlobalRole; scopes?: Scope[] } >; - type GetPersonalProject = AuthenticatedRequest<{}, Project>; - - type ProjectRelationPayload = { userId: string; role: ProjectRole }; type ProjectRelationResponse = { id: string; email: string; @@ -429,18 +394,6 @@ export declare namespace ProjectRequest { relations: ProjectRelationResponse[]; scopes: Scope[]; }; - - type Get = AuthenticatedRequest<{ projectId: string }, {}>; - type Update = AuthenticatedRequest< - { projectId: string }, - {}, - { - name?: string; - relations?: ProjectRelationPayload[]; - icon?: { type: 'icon' | 'emoji'; value: string }; - } - >; - type Delete = AuthenticatedRequest<{ projectId: string }, {}, {}, { transferId?: string }>; } // ---------------------------------- diff --git a/packages/cli/src/scaling/job-processor.ts b/packages/cli/src/scaling/job-processor.ts index 2aff0787c4..18a6f6b39f 100644 --- a/packages/cli/src/scaling/job-processor.ts +++ b/packages/cli/src/scaling/job-processor.ts @@ -1,6 +1,12 @@ import type { RunningJobSummary } from '@n8n/api-types'; import { Service } from '@n8n/di'; -import { InstanceSettings, WorkflowExecute, ErrorReporter, Logger } from 'n8n-core'; +import { + WorkflowHasIssuesError, + InstanceSettings, + WorkflowExecute, + ErrorReporter, + Logger, +} from 'n8n-core'; import type { ExecutionStatus, IExecuteResponsePromiseData, @@ -13,6 +19,7 @@ import type PCancelable from 'p-cancelable'; import config from '@/config'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import { getLifecycleHooksForScalingWorker } from '@/execution-lifecycle/execution-lifecycle-hooks'; import { ManualExecutionService } from '@/manual-execution.service'; import { NodeTypes } from '@/node-types'; import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; @@ -124,30 +131,29 @@ export class JobProcessor { const { pushRef } = job.data; - additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( + const lifecycleHooks = getLifecycleHooksForScalingWorker( execution.mode, job.data.executionId, execution.workflowData, - { retryOf: execution.retryOf as string, pushRef }, + { retryOf: execution.retryOf ?? undefined, pushRef }, ); + additionalData.hooks = lifecycleHooks; if (pushRef) { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment additionalData.sendDataToUI = WorkflowExecuteAdditionalData.sendDataToUI.bind({ pushRef }); } - additionalData.hooks.hookFunctions.sendResponse = [ - async (response: IExecuteResponsePromiseData): Promise => { - const msg: RespondToWebhookMessage = { - kind: 'respond-to-webhook', - executionId, - response: this.encodeWebhookResponse(response), - workerId: this.instanceSettings.hostId, - }; + lifecycleHooks.addHandler('sendResponse', async (response): Promise => { + const msg: RespondToWebhookMessage = { + kind: 'respond-to-webhook', + executionId, + response: this.encodeWebhookResponse(response), + workerId: this.instanceSettings.hostId, + }; - await job.progress(msg); - }, - ]; + await job.progress(msg); + }); additionalData.executionId = executionId; @@ -177,13 +183,33 @@ export class JobProcessor { userId: manualData?.userId, }; - workflowRun = this.manualExecutionService.runManually( - data, - workflow, - additionalData, - executionId, - resultData.pinData, - ); + try { + workflowRun = this.manualExecutionService.runManually( + data, + workflow, + additionalData, + executionId, + resultData.pinData, + ); + } catch (error) { + if (error instanceof WorkflowHasIssuesError) { + // execution did not even start, but we call `workflowExecuteAfter` to notify main + + const now = new Date(); + const runData: IRun = { + mode: 'manual', + status: 'error', + finished: false, + startedAt: now, + stoppedAt: now, + data: { resultData: { error, runData: {} } }, + }; + + await lifecycleHooks.runHook('workflowExecuteAfter', [runData]); + return { success: false }; + } + throw error; + } } else if (execution.data !== undefined) { workflowExecute = new WorkflowExecute(additionalData, execution.mode, execution.data); workflowRun = workflowExecute.processRunExecutionData(workflow); @@ -202,7 +228,7 @@ export class JobProcessor { workflowName: execution.workflowData.name, mode: execution.mode, startedAt, - retryOf: execution.retryOf ?? '', + retryOf: execution.retryOf ?? undefined, status: execution.status, }; diff --git a/packages/cli/src/scaling/scaling.service.ts b/packages/cli/src/scaling/scaling.service.ts index 536e835c72..7c48ce57e2 100644 --- a/packages/cli/src/scaling/scaling.service.ts +++ b/packages/cli/src/scaling/scaling.service.ts @@ -1,6 +1,6 @@ import { GlobalConfig } from '@n8n/config'; import { Container, Service } from '@n8n/di'; -import { ErrorReporter, InstanceSettings, Logger } from 'n8n-core'; +import { ErrorReporter, InstanceSettings, isObjectLiteral, Logger } from 'n8n-core'; import { ApplicationError, BINARY_ENCODING, @@ -17,7 +17,6 @@ import config from '@/config'; import { HIGHEST_SHUTDOWN_PRIORITY, Time } from '@/constants'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { OnShutdown } from '@/decorators/on-shutdown'; -import { MaxStalledCountError } from '@/errors/max-stalled-count.error'; import { EventService } from '@/events/event.service'; import { OrchestrationService } from '@/services/orchestration.service'; import { assertNever } from '@/utils'; @@ -93,6 +92,12 @@ export class ScalingService { void this.queue.process(JOB_TYPE_NAME, concurrency, async (job: Job) => { try { + if (!this.hasValidJobData(job)) { + throw new ApplicationError('Worker received invalid job', { + extra: { jobData: jsonStringify(job, { replaceCircularRefs: true }) }, + }); + } + await this.jobProcessor.processJob(job); } catch (error) { await this.reportJobProcessingError(ensureError(error), job); @@ -265,10 +270,6 @@ export class ScalingService { this.queue.on('error', (error: Error) => { if ('code' in error && error.code === 'ECONNREFUSED') return; // handled by RedisClientService.retryStrategy - if (error.message.includes('job stalled more than maxStalledCount')) { - throw new MaxStalledCountError(error); - } - /** * Non-recoverable error on worker start with Redis unavailable. * Even if Redis recovers, worker will remain unable to process jobs. @@ -503,5 +504,9 @@ export class ScalingService { : jsonStringify(error, { replaceCircularRefs: true }); } + private hasValidJobData(job: Job) { + return isObjectLiteral(job.data) && 'executionId' in job.data && 'loadStaticData' in job.data; + } + // #endregion } diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 09e3774aae..e505951dd4 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -37,7 +37,6 @@ import '@/controllers/active-workflows.controller'; import '@/controllers/annotation-tags.controller.ee'; import '@/controllers/auth.controller'; import '@/controllers/binary-data.controller'; -import '@/controllers/curl.controller'; import '@/controllers/ai.controller'; import '@/controllers/dynamic-node-parameters.controller'; import '@/controllers/invitation.controller'; @@ -135,6 +134,10 @@ export class Server extends AbstractServer { await import('@/controllers/cta.controller'); } + if (!this.globalConfig.tags.disabled) { + await import('@/controllers/tags.controller'); + } + // ---------------------------------------- // SAML // ---------------------------------------- @@ -319,8 +322,27 @@ export class Server extends AbstractServer { res.sendStatus(404); }; + const serveSchemas: express.RequestHandler = async (req, res) => { + const { node, version, resource, operation } = req.params; + const filePath = this.loadNodesAndCredentials.resolveSchema({ + node, + resource, + operation, + version, + }); + + if (filePath) { + try { + await fsAccess(filePath); + return res.sendFile(filePath, cacheOptions); + } catch {} + } + res.sendStatus(404); + }; + this.app.use('/icons/@:scope/:packageName/*/*.(svg|png)', serveIcons); this.app.use('/icons/:packageName/*/*.(svg|png)', serveIcons); + this.app.use('/schemas/:node/:version/:resource?/:operation?.json', serveSchemas); const isTLSEnabled = this.globalConfig.protocol === 'https' && !!(this.sslKey && this.sslCert); @@ -369,6 +391,7 @@ export class Server extends AbstractServer { method === 'GET' && accept && (accept.includes('text/html') || accept.includes('*/*')) && + !req.path.endsWith('.wasm') && !nonUIRoutesRegex.test(req.path) ) { res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); diff --git a/packages/cli/src/services/__tests__/curl.service.test.ts b/packages/cli/src/services/__tests__/curl.service.test.ts deleted file mode 100644 index 21f87b45b7..0000000000 --- a/packages/cli/src/services/__tests__/curl.service.test.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { CurlService } from '@/services/curl.service'; - -describe('CurlService', () => { - const service = new CurlService(); - - test('Should parse form-urlencoded content type correctly', () => { - const curl = - 'curl -X POST https://reqbin.com/echo/post/form -H "Content-Type: application/x-www-form-urlencoded" -d "param1=value1¶m2=value2"'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo/post/form'); - expect(parameters.sendBody).toBe(true); - expect(parameters.bodyParameters?.parameters[0].name).toBe('param1'); - expect(parameters.bodyParameters?.parameters[0].value).toBe('value1'); - expect(parameters.bodyParameters?.parameters[1].name).toBe('param2'); - expect(parameters.bodyParameters?.parameters[1].value).toBe('value2'); - expect(parameters.contentType).toBe('form-urlencoded'); - expect(parameters.sendHeaders).toBe(false); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse JSON content type correctly', () => { - const curl = - 'curl -X POST https://reqbin.com/echo/post/json -H \'Content-Type: application/json\' -d \'{"login":"my_login","password":"my_password"}\''; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo/post/json'); - expect(parameters.sendBody).toBe(true); - expect(parameters.bodyParameters?.parameters[0].name).toBe('login'); - expect(parameters.bodyParameters?.parameters[0].value).toBe('my_login'); - expect(parameters.bodyParameters?.parameters[1].name).toBe('password'); - expect(parameters.bodyParameters?.parameters[1].value).toBe('my_password'); - expect(parameters.contentType).toBe('json'); - expect(parameters.sendHeaders).toBe(false); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse multipart-form-data content type correctly', () => { - const curl = - 'curl -X POST https://reqbin.com/echo/post/json -v -F key1=value1 -F upload=@localfilename'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo/post/json'); - expect(parameters.sendBody).toBe(true); - expect(parameters.bodyParameters?.parameters[0].parameterType).toBe('formData'); - expect(parameters.bodyParameters?.parameters[0].name).toBe('key1'); - expect(parameters.bodyParameters?.parameters[0].value).toBe('value1'); - expect(parameters.bodyParameters?.parameters[1].parameterType).toBe('formBinaryData'); - expect(parameters.bodyParameters?.parameters[1].name).toBe('upload'); - expect(parameters.contentType).toBe('multipart-form-data'); - expect(parameters.sendHeaders).toBe(false); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse binary request correctly', () => { - const curl = - "curl --location --request POST 'https://www.website.com' --header 'Content-Type: image/png' --data-binary '@/Users/image.png"; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://www.website.com'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(true); - expect(parameters.contentType).toBe('binaryData'); - expect(parameters.sendHeaders).toBe(false); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse unknown content type correctly', () => { - const curl = `curl -X POST https://reqbin.com/echo/post/xml - -H "Content-Type: application/xml" - -H "Accept: application/xml" - -d "my_loginmy_password"`; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo/post/xml'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(true); - expect(parameters.contentType).toBe('raw'); - expect(parameters.rawContentType).toBe('application/xml'); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('Accept'); - expect(parameters.headerParameters?.parameters[0].value).toBe('application/xml'); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse header properties and keep the original case', () => { - const curl = - 'curl -X POST https://reqbin.com/echo/post/json -v -F key1=value1 -F upload=@localfilename -H "ACCEPT: text/javascript" -H "content-type: multipart/form-data"'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo/post/json'); - expect(parameters.sendBody).toBe(true); - expect(parameters.bodyParameters?.parameters[0].parameterType).toBe('formData'); - expect(parameters.bodyParameters?.parameters[0].name).toBe('key1'); - expect(parameters.bodyParameters?.parameters[0].value).toBe('value1'); - expect(parameters.bodyParameters?.parameters[1].parameterType).toBe('formBinaryData'); - expect(parameters.bodyParameters?.parameters[1].name).toBe('upload'); - expect(parameters.contentType).toBe('multipart-form-data'); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('ACCEPT'); - expect(parameters.headerParameters?.parameters[0].value).toBe('text/javascript'); - expect(parameters.sendQuery).toBe(false); - }); - - test('Should parse querystring properties', () => { - const curl = "curl -G -d 'q=kitties' -d 'count=20' https://google.com/search"; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://google.com/search'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendHeaders).toBe(false); - expect(parameters.sendQuery).toBe(true); - expect(parameters.queryParameters?.parameters[0].name).toBe('q'); - expect(parameters.queryParameters?.parameters[0].value).toBe('kitties'); - expect(parameters.queryParameters?.parameters[1].name).toBe('count'); - expect(parameters.queryParameters?.parameters[1].value).toBe('20'); - }); - - test('Should parse basic authentication property and keep the original case', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password"'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - }); - - test('Should parse location flag with --location', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" --location'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.redirect.redirect.followRedirects).toBe(true); - }); - - test('Should parse location flag with --L', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" -L'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.redirect.redirect.followRedirects).toBe(true); - }); - - test('Should parse location and max redirects flags with --location and --max-redirs 10', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" --location --max-redirs 10'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.redirect.redirect.followRedirects).toBe(true); - expect(parameters.options.redirect.redirect.maxRedirects).toBe('10'); - }); - - test('Should parse proxy flag -x', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" -x https://google.com'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.proxy).toBe('https://google.com'); - }); - - test('Should parse proxy flag --proxy', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" -x https://google.com'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.proxy).toBe('https://google.com'); - }); - - test('Should parse include headers on output flag --include', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" --include -x https://google.com'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.response.response.fullResponse).toBe(true); - }); - - test('Should parse include headers on output flag -i', () => { - const curl = 'curl https://reqbin.com/echo -u "login:password" -x https://google.com -i'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.sendBody).toBe(false); - expect(parameters.contentType).toBeUndefined(); - expect(parameters.sendQuery).toBe(false); - expect(parameters.sendHeaders).toBe(true); - expect(parameters.headerParameters?.parameters[0].name).toBe('authorization'); - expect(parameters.headerParameters?.parameters[0].value).toBe( - `Basic ${Buffer.from('login:password').toString('base64')}`, - ); - expect(parameters.options.response.response.fullResponse).toBe(true); - }); - - test('Should parse include request flag -X', () => { - const curl = 'curl -X POST https://reqbin.com/echo -u "login:password" -x https://google.com'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - }); - - test('Should parse include request flag --request', () => { - const curl = - 'curl --request POST https://reqbin.com/echo -u "login:password" -x https://google.com'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - }); - - test('Should parse include timeout flag --connect-timeout', () => { - const curl = - 'curl --request POST https://reqbin.com/echo -u "login:password" --connect-timeout 20'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - expect(parameters.options.timeout).toBe(20000); - }); - - test('Should parse download file flag -O', () => { - const curl = 'curl --request POST https://reqbin.com/echo -u "login:password" -O'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - expect(parameters.options.response.response.responseFormat).toBe('file'); - expect(parameters.options.response.response.outputPropertyName).toBe('data'); - }); - - test('Should parse download file flag -o', () => { - const curl = 'curl --request POST https://reqbin.com/echo -u "login:password" -o'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - expect(parameters.options.response.response.responseFormat).toBe('file'); - expect(parameters.options.response.response.outputPropertyName).toBe('data'); - }); - - test('Should parse ignore SSL flag -k', () => { - const curl = 'curl --request POST https://reqbin.com/echo -u "login:password" -k'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - expect(parameters.options.allowUnauthorizedCerts).toBe(true); - }); - - test('Should parse ignore SSL flag --insecure', () => { - const curl = 'curl --request POST https://reqbin.com/echo -u "login:password" --insecure'; - const parameters = service.toHttpNodeParameters(curl); - expect(parameters.url).toBe('https://reqbin.com/echo'); - expect(parameters.method).toBe('POST'); - expect(parameters.sendBody).toBe(false); - expect(parameters.options.allowUnauthorizedCerts).toBe(true); - }); -}); diff --git a/packages/cli/src/services/__tests__/public-api-key.service.test.ts b/packages/cli/src/services/__tests__/public-api-key.service.test.ts index 7c60b62983..86db071f35 100644 --- a/packages/cli/src/services/__tests__/public-api-key.service.test.ts +++ b/packages/cli/src/services/__tests__/public-api-key.service.test.ts @@ -144,4 +144,28 @@ describe('PublicApiKeyService', () => { ); }); }); + + describe('redactApiKey', () => { + it('should redact api key', async () => { + //Arrange + + const jwt = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE0ODUxNDA5ODQsImlhdCI6MTQ4NTEzNzM4NCwiaXNzIjoiYWNtZS5jb20iLCJzdWIiOiIyOWFjMGMxOC0wYjRhLTQyY2YtODJmYy0wM2Q1NzAzMThhMWQiLCJhcHBsaWNhdGlvbklkIjoiNzkxMDM3MzQtOTdhYi00ZDFhLWFmMzctZTAwNmQwNWQyOTUyIiwicm9sZXMiOltdfQ.Mp0Pcwsz5VECK11Kf2ZZNF_SMKu5CgBeLN9ZOP04kZo'; + + const publicApiKeyService = new PublicApiKeyService( + apiKeyRepository, + userRepository, + jwtService, + eventService, + ); + + //Act + + const redactedApiKey = publicApiKeyService.redactApiKey(jwt); + + //Assert + + expect(redactedApiKey).toBe('******4kZo'); + }); + }); }); diff --git a/packages/cli/src/services/credentials-tester.service.ts b/packages/cli/src/services/credentials-tester.service.ts index 1954923481..a45443a3bc 100644 --- a/packages/cli/src/services/credentials-tester.service.ts +++ b/packages/cli/src/services/credentials-tester.service.ts @@ -6,9 +6,10 @@ import { Service } from '@n8n/di'; import get from 'lodash/get'; import { + CredentialTestContext, ErrorReporter, + ExecuteContext, Logger, - NodeExecuteFunctions, RoutingNode, isObjectLiteral, } from 'n8n-core'; @@ -29,6 +30,7 @@ import type { INodeTypes, ICredentialTestFunctions, IDataObject, + IExecuteData, } from 'n8n-workflow'; import { VersionedNodeType, NodeHelpers, Workflow, ApplicationError } from 'n8n-workflow'; @@ -205,9 +207,8 @@ export class CredentialsTester { if (typeof credentialTestFunction === 'function') { // The credentials get tested via a function that is defined on the node - const credentialTestFunctions = NodeExecuteFunctions.getCredentialTestFunctions(); - - return credentialTestFunction.call(credentialTestFunctions, credentialsDecrypted); + const context = new CredentialTestContext(); + return credentialTestFunction.call(context, credentialsDecrypted); } // Credentials get tested via request instructions @@ -293,25 +294,24 @@ export class CredentialsTester { const additionalData = await WorkflowExecuteAdditionalData.getBase(user.id, node.parameters); - const routingNode = new RoutingNode( + const executeData: IExecuteData = { node, data: {}, source: null }; + const executeFunctions = new ExecuteContext( workflow, node, - connectionInputData, - runExecutionData ?? null, additionalData, mode, + runExecutionData, + runIndex, + connectionInputData, + inputData, + executeData, + [], ); + const routingNode = new RoutingNode(executeFunctions, nodeTypeCopy, credentialsDecrypted); let response: INodeExecutionData[][] | null | undefined; - try { - response = await routingNode.runNode( - inputData, - runIndex, - nodeTypeCopy, - { node, data: {}, source: null }, - credentialsDecrypted, - ); + response = await routingNode.runNode(); } catch (error) { this.errorReporter.error(error); // Do not fail any requests to allow custom error messages and diff --git a/packages/cli/src/services/curl.service.ts b/packages/cli/src/services/curl.service.ts deleted file mode 100644 index fae967154e..0000000000 --- a/packages/cli/src/services/curl.service.ts +++ /dev/null @@ -1,479 +0,0 @@ -import { Service } from '@n8n/di'; -import get from 'lodash/get'; -import type { IDataObject } from 'n8n-workflow'; -import { jsonParse } from 'n8n-workflow'; - -import curlconverter from 'curlconverter'; - -interface CurlJson { - url: string; - raw_url?: string; - method: string; - contentType?: string; - cookies?: { - [key: string]: string; - }; - auth?: { - user: string; - password: string; - }; - headers?: { - [key: string]: string; - }; - files?: { - [key: string]: string; - }; - queries: { - [key: string]: string; - }; - data?: { - [key: string]: string; - }; -} - -interface Parameter { - parameterType?: string; - name: string; - value: string; -} - -interface HttpNodeParameters { - url?: string; - method: string; - sendBody?: boolean; - authentication: string; - contentType?: 'form-urlencoded' | 'multipart-form-data' | 'json' | 'raw' | 'binaryData'; - rawContentType?: string; - specifyBody?: 'json' | 'keypair'; - bodyParameters?: { - parameters: Parameter[]; - }; - jsonBody?: object; - options: { - allowUnauthorizedCerts?: boolean; - proxy?: string; - timeout?: number; - redirect: { - redirect: { - followRedirects?: boolean; - maxRedirects?: number; - }; - }; - response: { - response: { - fullResponse?: boolean; - responseFormat?: string; - outputPropertyName?: string; - }; - }; - }; - sendHeaders?: boolean; - headerParameters?: { - parameters: Parameter[]; - }; - sendQuery?: boolean; - queryParameters?: { - parameters: Parameter[]; - }; -} - -type HttpNodeHeaders = Pick; - -type HttpNodeQueries = Pick; - -const enum ContentTypes { - applicationJson = 'application/json', - applicationFormUrlEncoded = 'application/x-www-form-urlencoded', - applicationMultipart = 'multipart/form-data', -} - -const SUPPORTED_CONTENT_TYPES = [ - ContentTypes.applicationJson, - ContentTypes.applicationFormUrlEncoded, - ContentTypes.applicationMultipart, -]; - -const CONTENT_TYPE_KEY = 'content-type'; - -const FOLLOW_REDIRECT_FLAGS = ['--location', '-L']; - -const MAX_REDIRECT_FLAG = '--max-redirs'; - -const PROXY_FLAGS = ['-x', '--proxy']; - -const INCLUDE_HEADERS_IN_OUTPUT_FLAGS = ['-i', '--include']; - -const REQUEST_FLAGS = ['-X', '--request']; - -const TIMEOUT_FLAGS = ['--connect-timeout']; - -const DOWNLOAD_FILE_FLAGS = ['-O', '-o']; - -const IGNORE_SSL_ISSUES_FLAGS = ['-k', '--insecure']; - -const isContentType = (headers: CurlJson['headers'], contentType: ContentTypes): boolean => { - return get(headers, CONTENT_TYPE_KEY) === contentType; -}; - -const isJsonRequest = (curlJson: CurlJson): boolean => { - if (isContentType(curlJson.headers, ContentTypes.applicationJson)) return true; - - if (curlJson.data) { - const bodyKey = Object.keys(curlJson.data)[0]; - try { - JSON.parse(bodyKey); - return true; - } catch { - return false; - } - } - return false; -}; - -const isFormUrlEncodedRequest = (curlJson: CurlJson): boolean => { - if (isContentType(curlJson.headers, ContentTypes.applicationFormUrlEncoded)) return true; - if (curlJson.data && !curlJson.files) return true; - return false; -}; - -const isMultipartRequest = (curlJson: CurlJson): boolean => { - if (isContentType(curlJson.headers, ContentTypes.applicationMultipart)) return true; - - // only multipart/form-data request include files - if (curlJson.files) return true; - return false; -}; - -const isBinaryRequest = (curlJson: CurlJson): boolean => { - if (curlJson?.headers?.[CONTENT_TYPE_KEY]) { - const contentType = curlJson?.headers?.[CONTENT_TYPE_KEY]; - return ['image', 'video', 'audio'].some((d) => contentType.includes(d)); - } - return false; -}; - -const sanitizeCurlCommand = (curlCommand: string) => - curlCommand - .replace(/\r\n/g, ' ') - .replace(/\n/g, ' ') - .replace(/\\/g, ' ') - .replace(/[ ]{2,}/g, ' '); - -const toKeyValueArray = ([key, value]: string[]) => ({ name: key, value }); - -const extractHeaders = (headers: CurlJson['headers'] = {}): HttpNodeHeaders => { - const emptyHeaders = !Object.keys(headers).length; - - const onlyContentTypeHeaderDefined = - Object.keys(headers).length === 1 && headers[CONTENT_TYPE_KEY] !== undefined; - - if (emptyHeaders || onlyContentTypeHeaderDefined) return { sendHeaders: false }; - - return { - sendHeaders: true, - headerParameters: { - parameters: Object.entries(headers) - .map(toKeyValueArray) - .filter((parameter) => parameter.name !== CONTENT_TYPE_KEY), - }, - }; -}; - -const extractQueries = (queries: CurlJson['queries'] = {}): HttpNodeQueries => { - const emptyQueries = !Object.keys(queries).length; - - if (emptyQueries) return { sendQuery: false }; - - return { - sendQuery: true, - queryParameters: { - parameters: Object.entries(queries).map(toKeyValueArray), - }, - }; -}; - -const extractJson = (body: CurlJson['data']) => - jsonParse<{ [key: string]: string }>(Object.keys(body as IDataObject)[0]); - -const jsonBodyToNodeParameters = (body: CurlJson['data'] = {}): Parameter[] | [] => { - const data = extractJson(body); - return Object.entries(data).map(toKeyValueArray); -}; - -const multipartToNodeParameters = ( - body: CurlJson['data'] = {}, - files: CurlJson['files'] = {}, -): Parameter[] | [] => { - return [ - ...Object.entries(body) - .map(toKeyValueArray) - .map((e) => ({ parameterType: 'formData', ...e })), - ...Object.entries(files) - .map(toKeyValueArray) - .map((e) => ({ parameterType: 'formBinaryData', ...e })), - ]; -}; - -const keyValueBodyToNodeParameters = (body: CurlJson['data'] = {}): Parameter[] | [] => { - return Object.entries(body).map(toKeyValueArray); -}; - -const lowerCaseContentTypeKey = (obj: { [x: string]: string }): void => { - const regex = new RegExp(CONTENT_TYPE_KEY, 'gi'); - - const contentTypeKey = Object.keys(obj).find((key) => { - const group = Array.from(key.matchAll(regex)); - if (group.length) return true; - return false; - }); - - if (!contentTypeKey) return; - - const value = obj[contentTypeKey]; - delete obj[contentTypeKey]; - obj[CONTENT_TYPE_KEY] = value; -}; - -const encodeBasicAuthentication = (username: string, password: string) => - Buffer.from(`${username}:${password}`).toString('base64'); - -const jsonHasNestedObjects = (json: { [key: string]: string | number | object }) => - Object.values(json).some((e) => typeof e === 'object'); - -const extractGroup = (curlCommand: string, regex: RegExp) => curlCommand.matchAll(regex); - -const mapCookies = (cookies: CurlJson['cookies']): { cookie: string } | {} => { - if (!cookies) return {}; - - const cookiesValues = Object.entries(cookies).reduce( - (accumulator: string, entry: [string, string]) => { - accumulator += `${entry[0]}=${entry[1]};`; - return accumulator; - }, - '', - ); - - if (!cookiesValues) return {}; - - return { - cookie: cookiesValues, - }; -}; - -export const flattenObject = (obj: { [x: string]: any }, prefix = '') => - Object.keys(obj).reduce((acc, k) => { - const pre = prefix.length ? prefix + '.' : ''; - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - if (typeof obj[k] === 'object') Object.assign(acc, flattenObject(obj[k], pre + k)); - //@ts-ignore - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - else acc[pre + k] = obj[k]; - return acc; - }, {}); - -@Service() -export class CurlService { - // eslint-disable-next-line complexity - toHttpNodeParameters(curlCommand: string): HttpNodeParameters { - const curlJson = jsonParse(curlconverter.toJsonString(curlCommand)); - - if (!curlJson.headers) curlJson.headers = {}; - - lowerCaseContentTypeKey(curlJson.headers); - - // set basic authentication - if (curlJson.auth) { - const { user, password: pass } = curlJson.auth; - Object.assign(curlJson.headers, { - authorization: `Basic ${encodeBasicAuthentication(user, pass)}`, - }); - } - - const httpNodeParameters: HttpNodeParameters = { - url: curlJson.url, - authentication: 'none', - method: curlJson.method.toUpperCase(), - ...extractHeaders({ ...curlJson.headers, ...mapCookies(curlJson.cookies) }), - ...extractQueries(curlJson.queries), - options: { - redirect: { - redirect: {}, - }, - response: { - response: {}, - }, - }, - }; - - //attempt to get the curl flags not supported by the library - const curl = sanitizeCurlCommand(curlCommand); - - //check for follow redirect flags - if (FOLLOW_REDIRECT_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - Object.assign(httpNodeParameters.options.redirect?.redirect, { followRedirects: true }); - - if (curl.includes(` ${MAX_REDIRECT_FLAG}`)) { - const extractedValue = Array.from( - extractGroup(curl, new RegExp(` ${MAX_REDIRECT_FLAG} (\\d+)`, 'g')), - ); - if (extractedValue.length) { - const [_, maxRedirects] = extractedValue[0]; - if (maxRedirects) { - Object.assign(httpNodeParameters.options.redirect?.redirect, { maxRedirects }); - } - } - } - } - - //check for proxy flags - if (PROXY_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - const foundFlag = PROXY_FLAGS.find((flag) => curl.includes(` ${flag}`)); - if (foundFlag) { - const extractedValue = Array.from( - extractGroup(curl, new RegExp(` ${foundFlag} (\\S*)`, 'g')), - ); - if (extractedValue.length) { - const [_, proxy] = extractedValue[0]; - Object.assign(httpNodeParameters.options, { proxy }); - } - } - } - - // check for "include header in output" flag - if (INCLUDE_HEADERS_IN_OUTPUT_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - Object.assign(httpNodeParameters.options?.response?.response, { - fullResponse: true, - responseFormat: 'autodetect', - }); - } - - // check for request flag - if (REQUEST_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - const foundFlag = REQUEST_FLAGS.find((flag) => curl.includes(` ${flag}`)); - if (foundFlag) { - const extractedValue = Array.from( - extractGroup(curl, new RegExp(` ${foundFlag} (\\w+)`, 'g')), - ); - if (extractedValue.length) { - const [_, request] = extractedValue[0]; - httpNodeParameters.method = request.toUpperCase(); - } - } - } - - // check for timeout flag - if (TIMEOUT_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - const foundFlag = TIMEOUT_FLAGS.find((flag) => curl.includes(` ${flag}`)); - if (foundFlag) { - const extractedValue = Array.from( - extractGroup(curl, new RegExp(` ${foundFlag} (\\d+)`, 'g')), - ); - if (extractedValue.length) { - const [_, timeout] = extractedValue[0]; - Object.assign(httpNodeParameters.options, { - timeout: parseInt(timeout, 10) * 1000, - }); - } - } - } - - // check for download flag - if (DOWNLOAD_FILE_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - const foundFlag = DOWNLOAD_FILE_FLAGS.find((flag) => curl.includes(` ${flag}`)); - if (foundFlag) { - Object.assign(httpNodeParameters.options.response.response, { - responseFormat: 'file', - outputPropertyName: 'data', - }); - } - } - - if (IGNORE_SSL_ISSUES_FLAGS.some((flag) => curl.includes(` ${flag}`))) { - const foundFlag = IGNORE_SSL_ISSUES_FLAGS.find((flag) => curl.includes(` ${flag}`)); - if (foundFlag) { - Object.assign(httpNodeParameters.options, { - allowUnauthorizedCerts: true, - }); - } - } - - const contentType = curlJson?.headers?.[CONTENT_TYPE_KEY] as ContentTypes; - - if (isBinaryRequest(curlJson)) { - return Object.assign(httpNodeParameters, { - contentType: 'binaryData', - sendBody: true, - }); - } - - if (contentType && !SUPPORTED_CONTENT_TYPES.includes(contentType)) { - return Object.assign(httpNodeParameters, { - sendBody: true, - contentType: 'raw', - rawContentType: contentType, - body: Object.keys(curlJson?.data ?? {})[0], - }); - } - - if (isJsonRequest(curlJson)) { - Object.assign(httpNodeParameters, { - contentType: 'json', - sendBody: true, - }); - - if (curlJson.data) { - const json = extractJson(curlJson.data); - - if (jsonHasNestedObjects(json)) { - // json body - Object.assign(httpNodeParameters, { - specifyBody: 'json', - jsonBody: JSON.stringify(json, null, 2), - }); - } else { - // key-value body - Object.assign(httpNodeParameters, { - specifyBody: 'keypair', - bodyParameters: { - parameters: jsonBodyToNodeParameters(curlJson.data), - }, - }); - } - } - } else if (isFormUrlEncodedRequest(curlJson)) { - Object.assign(httpNodeParameters, { - contentType: 'form-urlencoded', - sendBody: true, - specifyBody: 'keypair', - bodyParameters: { - parameters: keyValueBodyToNodeParameters(curlJson.data), - }, - }); - } else if (isMultipartRequest(curlJson)) { - Object.assign(httpNodeParameters, { - contentType: 'multipart-form-data', - sendBody: true, - bodyParameters: { - parameters: multipartToNodeParameters(curlJson.data, curlJson.files), - }, - }); - } else { - // could not figure the content type so do not set the body - Object.assign(httpNodeParameters, { - sendBody: false, - }); - } - - if (!Object.keys(httpNodeParameters.options?.redirect.redirect).length) { - // @ts-ignore - delete httpNodeParameters.options.redirect; - } - - if (!Object.keys(httpNodeParameters.options.response.response).length) { - // @ts-ignore - delete httpNodeParameters.options.response; - } - - return httpNodeParameters; - } -} diff --git a/packages/cli/src/services/dynamic-node-parameters.service.ts b/packages/cli/src/services/dynamic-node-parameters.service.ts index 0cde1c8489..1cbaf42363 100644 --- a/packages/cli/src/services/dynamic-node-parameters.service.ts +++ b/packages/cli/src/services/dynamic-node-parameters.service.ts @@ -1,5 +1,5 @@ import { Service } from '@n8n/di'; -import { LoadOptionsContext, RoutingNode, LocalLoadOptionsContext } from 'n8n-core'; +import { LoadOptionsContext, RoutingNode, LocalLoadOptionsContext, ExecuteContext } from 'n8n-core'; import type { ILoadOptions, ILoadOptionsFunctions, @@ -19,6 +19,7 @@ import type { NodeParameterValueType, IDataObject, ILocalLoadOptionsFunctions, + IExecuteData, } from 'n8n-workflow'; import { Workflow, ApplicationError } from 'n8n-workflow'; @@ -103,17 +104,8 @@ export class DynamicNodeParametersService { const workflow = this.getWorkflow(nodeTypeAndVersion, currentNodeParameters, credentials); const node = workflow.nodes['Temp-Node']; - const routingNode = new RoutingNode( - workflow, - node, - connectionInputData, - runExecutionData ?? null, - additionalData, - mode, - ); - // Create copy of node-type with the single property we want to get the data off - const tempNode: INodeType = { + const tempNodeType: INodeType = { ...nodeType, ...{ description: { @@ -135,11 +127,25 @@ export class DynamicNodeParametersService { main: [[{ json: {} }]], }; - const optionsData = await routingNode.runNode(inputData, runIndex, tempNode, { + const executeData: IExecuteData = { node, source: null, data: {}, - }); + }; + const executeFunctions = new ExecuteContext( + workflow, + node, + additionalData, + mode, + runExecutionData, + runIndex, + connectionInputData, + inputData, + executeData, + [], + ); + const routingNode = new RoutingNode(executeFunctions, tempNodeType); + const optionsData = await routingNode.runNode(); if (optionsData?.length === 0) { return []; diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index 099ae2c935..609e30ec60 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -12,7 +12,6 @@ import config from '@/config'; import { inE2ETests, LICENSE_FEATURES, N8N_VERSION } from '@/constants'; import { CredentialTypes } from '@/credential-types'; import { CredentialsOverwrites } from '@/credentials-overwrites'; -import { getVariablesLimit } from '@/environments.ee/variables/environment-helpers'; import { getLdapLoginLabel } from '@/ldap.ee/helpers.ee'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; @@ -147,6 +146,7 @@ export class FrontendService { }, }, publicApi: { + apiKeysPerUserLimit: this.license.getApiKeysPerUserLimit(), enabled: isApiEnabled(), latestVersion: 1, path: this.globalConfig.publicApi.path, @@ -154,7 +154,7 @@ export class FrontendService { enabled: !this.globalConfig.publicApi.swaggerUiDisabled, }, }, - workflowTagsDisabled: config.getEnv('workflowTagsDisabled'), + workflowTagsDisabled: this.globalConfig.tags.disabled, logLevel: this.globalConfig.logging.level, hiringBannerEnabled: config.getEnv('hiringBanner.enabled'), aiAssistant: { @@ -234,6 +234,7 @@ export class FrontendService { }, betaFeatures: this.frontendConfig.betaFeatures, easyAIWorkflowOnboarded: false, + partialExecution: this.globalConfig.partialExecutions, }; } @@ -326,7 +327,7 @@ export class FrontendService { } if (this.license.isVariablesEnabled()) { - this.settings.variables.limit = getVariablesLimit(); + this.settings.variables.limit = this.license.getVariablesLimit(); } if (this.license.isWorkflowHistoryLicensed() && config.getEnv('workflowHistory.enabled')) { diff --git a/packages/cli/src/services/project.service.ee.ts b/packages/cli/src/services/project.service.ee.ts index aa93360287..9eb3f8efa1 100644 --- a/packages/cli/src/services/project.service.ee.ts +++ b/packages/cli/src/services/project.service.ee.ts @@ -1,3 +1,4 @@ +import type { CreateProjectDto, ProjectRole, ProjectType, UpdateProjectDto } from '@n8n/api-types'; import { Container, Service } from '@n8n/di'; import { type Scope } from '@n8n/permissions'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import @@ -7,10 +8,8 @@ import { In, Not } from '@n8n/typeorm'; import { ApplicationError } from 'n8n-workflow'; import { UNLIMITED_LICENSE_QUOTA } from '@/constants'; -import type { ProjectIcon, ProjectType } from '@/databases/entities/project'; import { Project } from '@/databases/entities/project'; import { ProjectRelation } from '@/databases/entities/project-relation'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { User } from '@/databases/entities/user'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; @@ -131,7 +130,7 @@ export class ProjectService { ); } else { for (const sharedCredential of ownedCredentials) { - await credentialsService.delete(sharedCredential.credentials); + await credentialsService.delete(user, sharedCredential.credentials.id); } } @@ -168,12 +167,7 @@ export class ProjectService { return await this.projectRelationRepository.getPersonalProjectOwners(projectIds); } - async createTeamProject( - name: string, - adminUser: User, - id?: string, - icon?: ProjectIcon, - ): Promise { + async createTeamProject(adminUser: User, data: CreateProjectDto): Promise { const limit = this.license.getTeamProjectLimit(); if ( limit !== UNLIMITED_LICENSE_QUOTA && @@ -183,12 +177,7 @@ export class ProjectService { } const project = await this.projectRepository.save( - this.projectRepository.create({ - id, - name, - icon, - type: 'team', - }), + this.projectRepository.create({ ...data, type: 'team' }), ); // Link admin @@ -198,20 +187,10 @@ export class ProjectService { } async updateProject( - name: string, projectId: string, - icon?: { type: 'icon' | 'emoji'; value: string }, + data: Pick, ): Promise { - const result = await this.projectRepository.update( - { - id: projectId, - type: 'team', - }, - { - name, - icon, - }, - ); + const result = await this.projectRepository.update({ id: projectId, type: 'team' }, data); if (!result.affected) { throw new ForbiddenError('Project not found'); diff --git a/packages/cli/src/services/public-api-key.service.ts b/packages/cli/src/services/public-api-key.service.ts index a6b1133bc2..719f922fb2 100644 --- a/packages/cli/src/services/public-api-key.service.ts +++ b/packages/cli/src/services/public-api-key.service.ts @@ -1,4 +1,7 @@ +import type { UnixTimestamp, UpdateApiKeyRequestDto } from '@n8n/api-types'; +import type { CreateApiKeyRequestDto } from '@n8n/api-types/src/dto/api-keys/create-api-key-request.dto'; import { Service } from '@n8n/di'; +import { TokenExpiredError } from 'jsonwebtoken'; import type { OpenAPIV3 } from 'openapi-types'; import { ApiKey } from '@/databases/entities/api-key'; @@ -12,8 +15,8 @@ import { JwtService } from './jwt.service'; const API_KEY_AUDIENCE = 'public-api'; const API_KEY_ISSUER = 'n8n'; -const REDACT_API_KEY_REVEAL_COUNT = 15; -const REDACT_API_KEY_MAX_LENGTH = 80; +const REDACT_API_KEY_REVEAL_COUNT = 4; +const REDACT_API_KEY_MAX_LENGTH = 10; @Service() export class PublicApiKeyService { @@ -27,17 +30,15 @@ export class PublicApiKeyService { /** * Creates a new public API key for the specified user. * @param user - The user for whom the API key is being created. - * @returns A promise that resolves to the newly created API key. */ - async createPublicApiKeyForUser(user: User) { - const apiKey = this.generateApiKey(user); - await this.apiKeyRepository.upsert( + async createPublicApiKeyForUser(user: User, { label, expiresAt }: CreateApiKeyRequestDto) { + const apiKey = this.generateApiKey(user, expiresAt); + await this.apiKeyRepository.insert( this.apiKeyRepository.create({ userId: user.id, apiKey, - label: 'My API Key', + label, }), - ['apiKey'], ); return await this.apiKeyRepository.findOneByOrFail({ apiKey }); @@ -46,13 +47,13 @@ export class PublicApiKeyService { /** * Retrieves and redacts API keys for a given user. * @param user - The user for whom to retrieve and redact API keys. - * @returns A promise that resolves to an array of objects containing redacted API keys. */ async getRedactedApiKeysForUser(user: User) { const apiKeys = await this.apiKeyRepository.findBy({ userId: user.id }); return apiKeys.map((apiKeyRecord) => ({ ...apiKeyRecord, apiKey: this.redactApiKey(apiKeyRecord.apiKey), + expiresAt: this.getApiKeyExpiration(apiKeyRecord.apiKey), })); } @@ -60,6 +61,10 @@ export class PublicApiKeyService { await this.apiKeyRepository.delete({ userId: user.id, id: apiKeyId }); } + async updateApiKeyForUser(user: User, apiKeyId: string, { label }: UpdateApiKeyRequestDto) { + await this.apiKeyRepository.update({ id: apiKeyId, userId: user.id }, { label }); + } + private async getUserForApiKey(apiKey: string) { return await this.userRepository .createQueryBuilder('user') @@ -70,22 +75,24 @@ export class PublicApiKeyService { } /** - * Redacts an API key by keeping the first few characters and replacing the rest with asterisks. - * @param apiKey - The API key to be redacted. If null, the function returns undefined. - * @returns The redacted API key with a fixed prefix and asterisks replacing the rest of the characters. + * Redacts an API key by replacing a portion of it with asterisks. + * + * The function keeps the last `REDACT_API_KEY_REVEAL_COUNT` characters of the API key visible + * and replaces the rest with asterisks, up to a maximum length defined by `REDACT_API_KEY_MAX_LENGTH`. + * * @example * ```typescript * const redactedKey = PublicApiKeyService.redactApiKey('12345-abcdef-67890'); - * console.log(redactedKey); // Output: '12345-*****' + * console.log(redactedKey); // Output: '*****-67890' * ``` */ redactApiKey(apiKey: string) { - const visiblePart = apiKey.slice(0, REDACT_API_KEY_REVEAL_COUNT); - const redactedPart = '*'.repeat(apiKey.length - REDACT_API_KEY_REVEAL_COUNT); + const visiblePart = apiKey.slice(-REDACT_API_KEY_REVEAL_COUNT); + const redactedPart = '*'.repeat( + Math.max(0, REDACT_API_KEY_MAX_LENGTH - REDACT_API_KEY_REVEAL_COUNT), + ); - const completeRedactedApiKey = visiblePart + redactedPart; - - return completeRedactedApiKey.slice(0, REDACT_API_KEY_MAX_LENGTH); + return redactedPart + visiblePart; } getAuthMiddleware(version: string) { @@ -100,6 +107,16 @@ export class PublicApiKeyService { if (!user) return false; + try { + this.jwtService.verify(providedApiKey, { + issuer: API_KEY_ISSUER, + audience: API_KEY_AUDIENCE, + }); + } catch (e) { + if (e instanceof TokenExpiredError) return false; + throw e; + } + this.eventService.emit('public-api-invoked', { userId: user.id, path: req.path, @@ -113,6 +130,17 @@ export class PublicApiKeyService { }; } - private generateApiKey = (user: User) => - this.jwtService.sign({ sub: user.id, iss: API_KEY_ISSUER, aud: API_KEY_AUDIENCE }); + private generateApiKey = (user: User, expiresAt: UnixTimestamp) => { + const nowInSeconds = Math.floor(Date.now() / 1000); + + return this.jwtService.sign( + { sub: user.id, iss: API_KEY_ISSUER, aud: API_KEY_AUDIENCE }, + { ...(expiresAt && { expiresIn: expiresAt - nowInSeconds }) }, + ); + }; + + private getApiKeyExpiration = (apiKey: string) => { + const decoded = this.jwtService.decode(apiKey); + return decoded?.exp ?? null; + }; } diff --git a/packages/cli/src/services/redis-client.service.ts b/packages/cli/src/services/redis-client.service.ts index 6dea89a801..e31d55fb6c 100644 --- a/packages/cli/src/services/redis-client.service.ts +++ b/packages/cli/src/services/redis-client.service.ts @@ -131,7 +131,7 @@ export class RedisClientService extends TypedEmitter { } private getOptions({ extraOptions }: { extraOptions?: RedisOptions }) { - const { username, password, db, tls } = this.globalConfig.queue.bull.redis; + const { username, password, db, tls, dualStack } = this.globalConfig.queue.bull.redis; /** * Disabling ready check allows quick reconnection to Redis if Redis becomes @@ -153,6 +153,8 @@ export class RedisClientService extends TypedEmitter { ...extraOptions, }; + if (dualStack) options.family = 0; + if (tls) options.tls = {}; // enable TLS with default Node.js settings return options; diff --git a/packages/cli/src/services/role.service.ts b/packages/cli/src/services/role.service.ts index 7590dca2d5..e060c0dd81 100644 --- a/packages/cli/src/services/role.service.ts +++ b/packages/cli/src/services/role.service.ts @@ -1,9 +1,10 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Service } from '@n8n/di'; import { combineScopes, type Resource, type Scope } from '@n8n/permissions'; import { ApplicationError } from 'n8n-workflow'; import type { CredentialsEntity } from '@/databases/entities/credentials-entity'; -import type { ProjectRelation, ProjectRole } from '@/databases/entities/project-relation'; +import type { ProjectRelation } from '@/databases/entities/project-relation'; import type { CredentialSharingRole, SharedCredentials, diff --git a/packages/cli/src/services/tag.service.ts b/packages/cli/src/services/tag.service.ts index 09695f44ff..1f6cdd88fe 100644 --- a/packages/cli/src/services/tag.service.ts +++ b/packages/cli/src/services/tag.service.ts @@ -8,6 +8,8 @@ import type { ITagWithCountDb } from '@/interfaces'; type GetAllResult = T extends { withUsageCount: true } ? ITagWithCountDb[] : TagEntity[]; +type Action = 'Create' | 'Update'; + @Service() export class TagService { constructor( @@ -24,7 +26,7 @@ export class TagService { async save(tag: TagEntity, actionKind: 'create' | 'update') { await validateEntity(tag); - const action = actionKind[0].toUpperCase() + actionKind.slice(1); + const action = (actionKind[0].toUpperCase() + actionKind.slice(1)) as Action; await this.externalHooks.run(`tag.before${action}`, [tag]); diff --git a/packages/cli/src/task-runners/__tests__/task-runner-process-restart-loop-detector.test.ts b/packages/cli/src/task-runners/__tests__/task-runner-process-restart-loop-detector.test.ts index bf3bab4c27..808cc1cb93 100644 --- a/packages/cli/src/task-runners/__tests__/task-runner-process-restart-loop-detector.test.ts +++ b/packages/cli/src/task-runners/__tests__/task-runner-process-restart-loop-detector.test.ts @@ -2,15 +2,15 @@ import { TaskRunnersConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; import type { Logger } from 'n8n-core'; -import type { TaskRunnerAuthService } from '@/task-runners/auth/task-runner-auth.service'; import { TaskRunnerRestartLoopError } from '@/task-runners/errors/task-runner-restart-loop-error'; +import type { TaskBrokerAuthService } from '@/task-runners/task-broker/auth/task-broker-auth.service'; import { TaskRunnerLifecycleEvents } from '@/task-runners/task-runner-lifecycle-events'; import { TaskRunnerProcess } from '@/task-runners/task-runner-process'; import { TaskRunnerProcessRestartLoopDetector } from '@/task-runners/task-runner-process-restart-loop-detector'; describe('TaskRunnerProcessRestartLoopDetector', () => { const mockLogger = mock(); - const mockAuthService = mock(); + const mockAuthService = mock(); const runnerConfig = new TaskRunnersConfig(); const taskRunnerProcess = new TaskRunnerProcess( mockLogger, diff --git a/packages/cli/src/task-runners/__tests__/task-runner-process.test.ts b/packages/cli/src/task-runners/__tests__/task-runner-process.test.ts index d00ce7b88f..0d08ecd1c2 100644 --- a/packages/cli/src/task-runners/__tests__/task-runner-process.test.ts +++ b/packages/cli/src/task-runners/__tests__/task-runner-process.test.ts @@ -3,7 +3,7 @@ import { mock } from 'jest-mock-extended'; import { Logger } from 'n8n-core'; import type { ChildProcess, SpawnOptions } from 'node:child_process'; -import type { TaskRunnerAuthService } from '@/task-runners/auth/task-runner-auth.service'; +import type { TaskBrokerAuthService } from '@/task-runners/task-broker/auth/task-broker-auth.service'; import { TaskRunnerProcess } from '@/task-runners/task-runner-process'; import { mockInstance } from '@test/mocking'; @@ -26,7 +26,7 @@ describe('TaskRunnerProcess', () => { const runnerConfig = mockInstance(TaskRunnersConfig); runnerConfig.enabled = true; runnerConfig.mode = 'internal'; - const authService = mock(); + const authService = mock(); let taskRunnerProcess = new TaskRunnerProcess(logger, runnerConfig, authService, mock()); afterEach(async () => { @@ -117,5 +117,17 @@ describe('TaskRunnerProcess', () => { const options = spawnMock.mock.calls[0][2] as SpawnOptions; expect(options.env).not.toHaveProperty('NODE_OPTIONS'); }); + + it('should use --disallow-code-generation-from-strings and --disable-proto=delete flags', async () => { + jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); + + await taskRunnerProcess.start(); + + expect(spawnMock.mock.calls[0].at(1)).toEqual([ + '--disallow-code-generation-from-strings', + '--disable-proto=delete', + expect.stringContaining('/packages/@n8n/task-runner/dist/start.js'), + ]); + }); }); }); diff --git a/packages/cli/src/task-runners/auth/task-runner-auth.schema.ts b/packages/cli/src/task-runners/auth/task-runner-auth.schema.ts deleted file mode 100644 index c3ab2c17f2..0000000000 --- a/packages/cli/src/task-runners/auth/task-runner-auth.schema.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { z } from 'zod'; - -export const taskRunnerAuthRequestBodySchema = z.object({ - token: z.string().min(1), -}); diff --git a/packages/cli/src/task-runners/default-task-runner-disconnect-analyzer.ts b/packages/cli/src/task-runners/default-task-runner-disconnect-analyzer.ts index a0193c40e0..1743c531a4 100644 --- a/packages/cli/src/task-runners/default-task-runner-disconnect-analyzer.ts +++ b/packages/cli/src/task-runners/default-task-runner-disconnect-analyzer.ts @@ -1,10 +1,13 @@ import { Service } from '@n8n/di'; import config from '@/config'; +import type { + DisconnectAnalyzer, + DisconnectErrorOptions, +} from '@/task-runners/task-broker/task-broker-types'; import { TaskRunnerDisconnectedError } from './errors/task-runner-disconnected-error'; import { TaskRunnerFailedHeartbeatError } from './errors/task-runner-failed-heartbeat.error'; -import type { DisconnectAnalyzer, DisconnectErrorOptions } from './task-runner-types'; /** * Analyzes the disconnect reason of a task runner to provide a more diff --git a/packages/cli/src/task-runners/errors/task-runner-oom-error.ts b/packages/cli/src/task-runners/errors/task-runner-oom-error.ts index 5c78bef816..412cf6e626 100644 --- a/packages/cli/src/task-runners/errors/task-runner-oom-error.ts +++ b/packages/cli/src/task-runners/errors/task-runner-oom-error.ts @@ -1,6 +1,6 @@ import { ApplicationError } from 'n8n-workflow'; -import type { TaskRunner } from '../task-broker.service'; +import type { TaskRunner } from '@/task-runners/task-broker/task-broker.service'; export class TaskRunnerOomError extends ApplicationError { description: string; diff --git a/packages/cli/src/task-runners/internal-task-runner-disconnect-analyzer.ts b/packages/cli/src/task-runners/internal-task-runner-disconnect-analyzer.ts index a84682e521..386cab23fd 100644 --- a/packages/cli/src/task-runners/internal-task-runner-disconnect-analyzer.ts +++ b/packages/cli/src/task-runners/internal-task-runner-disconnect-analyzer.ts @@ -1,12 +1,13 @@ import { TaskRunnersConfig } from '@n8n/config'; import { Service } from '@n8n/di'; +import type { DisconnectErrorOptions } from '@/task-runners/task-broker/task-broker-types'; + import { DefaultTaskRunnerDisconnectAnalyzer } from './default-task-runner-disconnect-analyzer'; import { TaskRunnerOomError } from './errors/task-runner-oom-error'; import { SlidingWindowSignal } from './sliding-window-signal'; import type { ExitReason, TaskRunnerProcessEventMap } from './task-runner-process'; import { TaskRunnerProcess } from './task-runner-process'; -import type { DisconnectErrorOptions } from './task-runner-types'; /** * Analyzes the disconnect reason of a task runner process to provide a more diff --git a/packages/cli/src/task-runners/__tests__/task-runner-server.test.ts b/packages/cli/src/task-runners/task-broker/__tests__/task-broker-server.test.ts similarity index 70% rename from packages/cli/src/task-runners/__tests__/task-runner-server.test.ts rename to packages/cli/src/task-runners/task-broker/__tests__/task-broker-server.test.ts index 33de18c605..a474bbecc0 100644 --- a/packages/cli/src/task-runners/__tests__/task-runner-server.test.ts +++ b/packages/cli/src/task-runners/task-broker/__tests__/task-broker-server.test.ts @@ -3,24 +3,23 @@ import { mock } from 'jest-mock-extended'; import { ServerResponse } from 'node:http'; import type WebSocket from 'ws'; -import type { TaskRunnerAuthController } from '@/task-runners/auth/task-runner-auth.controller'; -import { TaskRunnerServer } from '@/task-runners/task-runner-server'; +import type { TaskBrokerAuthController } from '@/task-runners/task-broker/auth/task-broker-auth.controller'; +import { TaskBrokerServer } from '@/task-runners/task-broker/task-broker-server'; +import type { TaskBrokerServerInitRequest } from '@/task-runners/task-broker/task-broker-types'; -import type { TaskRunnerServerInitRequest } from '../task-runner-types'; - -describe('TaskRunnerServer', () => { +describe('TaskBrokerServer', () => { describe('handleUpgradeRequest', () => { it('should close WebSocket when response status code is > 200', () => { const ws = mock(); - const request = mock({ + const request = mock({ url: '/runners/_ws', ws, }); - const server = new TaskRunnerServer( + const server = new TaskBrokerServer( mock(), mock({ taskRunners: { path: '/runners' } }), - mock(), + mock(), mock(), ); @@ -39,15 +38,15 @@ describe('TaskRunnerServer', () => { it('should not close WebSocket when response status code is 200', () => { const ws = mock(); - const request = mock({ + const request = mock({ url: '/runners/_ws', ws, }); - const server = new TaskRunnerServer( + const server = new TaskBrokerServer( mock(), mock({ taskRunners: { path: '/runners' } }), - mock(), + mock(), mock(), ); diff --git a/packages/cli/src/task-runners/__tests__/task-runner-ws-server.test.ts b/packages/cli/src/task-runners/task-broker/__tests__/task-broker-ws-server.test.ts similarity index 86% rename from packages/cli/src/task-runners/__tests__/task-runner-ws-server.test.ts rename to packages/cli/src/task-runners/task-broker/__tests__/task-broker-ws-server.test.ts index cabedc530b..2a4dc85b42 100644 --- a/packages/cli/src/task-runners/__tests__/task-runner-ws-server.test.ts +++ b/packages/cli/src/task-runners/task-broker/__tests__/task-broker-ws-server.test.ts @@ -3,12 +3,12 @@ import { mock } from 'jest-mock-extended'; import type WebSocket from 'ws'; import { Time, WsStatusCodes } from '@/constants'; -import { TaskRunnerWsServer } from '@/task-runners/task-runner-ws-server'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; -describe('TaskRunnerWsServer', () => { +describe('TaskBrokerWsServer', () => { describe('removeConnection', () => { it('should close with 1000 status code by default', async () => { - const server = new TaskRunnerWsServer(mock(), mock(), mock(), mock(), mock()); + const server = new TaskBrokerWsServer(mock(), mock(), mock(), mock(), mock()); const ws = mock(); server.runnerConnections.set('test-runner', ws); @@ -22,7 +22,7 @@ describe('TaskRunnerWsServer', () => { it('should set up heartbeat timer on server start', async () => { const setIntervalSpy = jest.spyOn(global, 'setInterval'); - const server = new TaskRunnerWsServer( + const server = new TaskBrokerWsServer( mock(), mock(), mock(), @@ -44,7 +44,7 @@ describe('TaskRunnerWsServer', () => { jest.spyOn(global, 'setInterval'); const clearIntervalSpy = jest.spyOn(global, 'clearInterval'); - const server = new TaskRunnerWsServer( + const server = new TaskBrokerWsServer( mock(), mock(), mock(), @@ -61,7 +61,7 @@ describe('TaskRunnerWsServer', () => { describe('sendMessage', () => { it('should work with a message containing circular references', () => { - const server = new TaskRunnerWsServer(mock(), mock(), mock(), mock(), mock()); + const server = new TaskBrokerWsServer(mock(), mock(), mock(), mock(), mock()); const ws = mock(); server.runnerConnections.set('test-runner', ws); diff --git a/packages/cli/src/task-runners/__tests__/task-broker.test.ts b/packages/cli/src/task-runners/task-broker/__tests__/task-broker.service.test.ts similarity index 99% rename from packages/cli/src/task-runners/__tests__/task-broker.test.ts rename to packages/cli/src/task-runners/task-broker/__tests__/task-broker.service.test.ts index ced7e1c07e..90261acdc9 100644 --- a/packages/cli/src/task-runners/__tests__/task-broker.test.ts +++ b/packages/cli/src/task-runners/task-broker/__tests__/task-broker.service.test.ts @@ -4,12 +4,12 @@ import { mock } from 'jest-mock-extended'; import { ApplicationError, type INodeTypeBaseDescription } from 'n8n-workflow'; import { Time } from '@/constants'; +import type { TaskRunnerLifecycleEvents } from '@/task-runners/task-runner-lifecycle-events'; -import { TaskRejectError } from '../errors'; +import { TaskRejectError } from '../errors/task-reject.error'; import { TaskRunnerTimeoutError } from '../errors/task-runner-timeout.error'; import { TaskBroker } from '../task-broker.service'; import type { TaskOffer, TaskRequest, TaskRunner } from '../task-broker.service'; -import type { TaskRunnerLifecycleEvents } from '../task-runner-lifecycle-events'; const createValidUntil = (ms: number) => process.hrtime.bigint() + BigInt(ms * 1_000_000); diff --git a/packages/cli/src/task-runners/auth/__tests__/task-runner-auth.controller.test.ts b/packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.controller.test.ts similarity index 80% rename from packages/cli/src/task-runners/auth/__tests__/task-runner-auth.controller.test.ts rename to packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.controller.test.ts index 3c650d1644..1ff547923c 100644 --- a/packages/cli/src/task-runners/auth/__tests__/task-runner-auth.controller.test.ts +++ b/packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.controller.test.ts @@ -5,14 +5,14 @@ import { mock } from 'jest-mock-extended'; import { CacheService } from '@/services/cache/cache.service'; import { mockInstance } from '@test/mocking'; -import { BadRequestError } from '../../../errors/response-errors/bad-request.error'; -import { ForbiddenError } from '../../../errors/response-errors/forbidden.error'; -import type { AuthlessRequest } from '../../../requests'; -import type { TaskRunnerServerInitRequest } from '../../task-runner-types'; -import { TaskRunnerAuthController } from '../task-runner-auth.controller'; -import { TaskRunnerAuthService } from '../task-runner-auth.service'; +import { BadRequestError } from '../../../../errors/response-errors/bad-request.error'; +import { ForbiddenError } from '../../../../errors/response-errors/forbidden.error'; +import type { AuthlessRequest } from '../../../../requests'; +import type { TaskBrokerServerInitRequest } from '../../task-broker-types'; +import { TaskBrokerAuthController } from '../task-broker-auth.controller'; +import { TaskBrokerAuthService } from '../task-broker-auth.service'; -describe('TaskRunnerAuthController', () => { +describe('TaskBrokerAuthController', () => { const globalConfig = mockInstance(GlobalConfig, { cache: { backend: 'memory', @@ -27,8 +27,8 @@ describe('TaskRunnerAuthController', () => { }); const TTL = 100; const cacheService = new CacheService(globalConfig); - const authService = new TaskRunnerAuthService(globalConfig, cacheService, TTL); - const authController = new TaskRunnerAuthController(authService); + const authService = new TaskBrokerAuthService(globalConfig, cacheService, TTL); + const authController = new TaskBrokerAuthController(authService); const createMockGrantTokenReq = (token?: string) => ({ @@ -71,7 +71,7 @@ describe('TaskRunnerAuthController', () => { const next = jest.fn() as NextFunction; const createMockReqWithToken = (token?: string) => - mock({ + mock({ headers: { authorization: `Bearer ${token}`, }, @@ -82,7 +82,7 @@ describe('TaskRunnerAuthController', () => { }); it('should respond with 401 when grant token is missing', async () => { - const req = mock({}); + const req = mock({}); await authController.authMiddleware(req, res, next); diff --git a/packages/cli/src/task-runners/auth/__tests__/task-runner-auth.service.test.ts b/packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.service.test.ts similarity index 86% rename from packages/cli/src/task-runners/auth/__tests__/task-runner-auth.service.test.ts rename to packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.service.test.ts index a1321945e3..4585f80b5b 100644 --- a/packages/cli/src/task-runners/auth/__tests__/task-runner-auth.service.test.ts +++ b/packages/cli/src/task-runners/task-broker/auth/__tests__/task-broker-auth.service.test.ts @@ -5,10 +5,10 @@ import config from '@/config'; import { CacheService } from '@/services/cache/cache.service'; import { retryUntil } from '@test-integration/retry-until'; -import { mockInstance } from '../../../../test/shared/mocking'; -import { TaskRunnerAuthService } from '../task-runner-auth.service'; +import { mockInstance } from '../../../../../test/shared/mocking'; +import { TaskBrokerAuthService } from '../task-broker-auth.service'; -describe('TaskRunnerAuthService', () => { +describe('TaskBrokerAuthService', () => { config.set('taskRunners.authToken', 'random-secret'); const globalConfig = mockInstance(GlobalConfig, { @@ -25,7 +25,7 @@ describe('TaskRunnerAuthService', () => { }); const TTL = 100; const cacheService = new CacheService(globalConfig); - const authService = new TaskRunnerAuthService(globalConfig, cacheService, TTL); + const authService = new TaskBrokerAuthService(globalConfig, cacheService, TTL); beforeEach(() => { jest.clearAllMocks(); @@ -33,11 +33,11 @@ describe('TaskRunnerAuthService', () => { describe('isValidAuthToken', () => { it('should be valid for the configured token', () => { - expect(authService.isValidAuthToken('random-secret')); + expect(authService.isValidAuthToken('random-secret')).toBe(true); }); it('should be invalid for anything else', () => { - expect(authService.isValidAuthToken('!random-secret')); + expect(authService.isValidAuthToken('!random-secret')).toBe(false); }); }); diff --git a/packages/cli/src/task-runners/auth/task-runner-auth.controller.ts b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.controller.ts similarity index 60% rename from packages/cli/src/task-runners/auth/task-runner-auth.controller.ts rename to packages/cli/src/task-runners/task-broker/auth/task-broker-auth.controller.ts index 4ea5f3b6f4..e9fd61c17b 100644 --- a/packages/cli/src/task-runners/auth/task-runner-auth.controller.ts +++ b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.controller.ts @@ -2,19 +2,19 @@ import { Service } from '@n8n/di'; import type { NextFunction, Response } from 'express'; import type { AuthlessRequest } from '@/requests'; +import type { TaskBrokerServerInitRequest } from '@/task-runners/task-broker/task-broker-types'; -import { taskRunnerAuthRequestBodySchema } from './task-runner-auth.schema'; -import { TaskRunnerAuthService } from './task-runner-auth.service'; -import { BadRequestError } from '../../errors/response-errors/bad-request.error'; -import { ForbiddenError } from '../../errors/response-errors/forbidden.error'; -import type { TaskRunnerServerInitRequest } from '../task-runner-types'; +import { taskBrokerAuthRequestBodySchema } from './task-broker-auth.schema'; +import { TaskBrokerAuthService } from './task-broker-auth.service'; +import { BadRequestError } from '../../../errors/response-errors/bad-request.error'; +import { ForbiddenError } from '../../../errors/response-errors/forbidden.error'; /** * Controller responsible for authenticating Task Runner connections */ @Service() -export class TaskRunnerAuthController { - constructor(private readonly taskRunnerAuthService: TaskRunnerAuthService) { +export class TaskBrokerAuthController { + constructor(private readonly authService: TaskBrokerAuthService) { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment this.authMiddleware = this.authMiddleware.bind(this); } @@ -24,17 +24,17 @@ export class TaskRunnerAuthController { * which can be used to initiate a task runner connection. */ async createGrantToken(req: AuthlessRequest) { - const result = await taskRunnerAuthRequestBodySchema.safeParseAsync(req.body); + const result = await taskBrokerAuthRequestBodySchema.safeParseAsync(req.body); if (!result.success) { throw new BadRequestError(result.error.errors[0].code); } const { token: authToken } = result.data; - if (!this.taskRunnerAuthService.isValidAuthToken(authToken)) { + if (!this.authService.isValidAuthToken(authToken)) { throw new ForbiddenError(); } - const grantToken = await this.taskRunnerAuthService.createGrantToken(); + const grantToken = await this.authService.createGrantToken(); return { token: grantToken, }; @@ -43,7 +43,7 @@ export class TaskRunnerAuthController { /** * Middleware to authenticate task runner init requests */ - async authMiddleware(req: TaskRunnerServerInitRequest, res: Response, next: NextFunction) { + async authMiddleware(req: TaskBrokerServerInitRequest, res: Response, next: NextFunction) { const authHeader = req.headers.authorization; if (typeof authHeader !== 'string' || !authHeader.startsWith('Bearer ')) { res.status(401).json({ code: 401, message: 'Unauthorized' }); @@ -51,7 +51,7 @@ export class TaskRunnerAuthController { } const grantToken = authHeader.slice('Bearer '.length); - const isConsumed = await this.taskRunnerAuthService.tryConsumeGrantToken(grantToken); + const isConsumed = await this.authService.tryConsumeGrantToken(grantToken); if (!isConsumed) { res.status(403).json({ code: 403, message: 'Forbidden' }); return; diff --git a/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.schema.ts b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.schema.ts new file mode 100644 index 0000000000..89c1acefb8 --- /dev/null +++ b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.schema.ts @@ -0,0 +1,5 @@ +import { z } from 'zod'; + +export const taskBrokerAuthRequestBodySchema = z.object({ + token: z.string().min(1), +}); diff --git a/packages/cli/src/task-runners/auth/task-runner-auth.service.ts b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.service.ts similarity index 80% rename from packages/cli/src/task-runners/auth/task-runner-auth.service.ts rename to packages/cli/src/task-runners/task-broker/auth/task-broker-auth.service.ts index c16afc5c63..1c401afed5 100644 --- a/packages/cli/src/task-runners/auth/task-runner-auth.service.ts +++ b/packages/cli/src/task-runners/task-broker/auth/task-broker-auth.service.ts @@ -1,6 +1,6 @@ import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; -import { randomBytes } from 'crypto'; +import { randomBytes, timingSafeEqual } from 'crypto'; import { Time } from '@/constants'; import { CacheService } from '@/services/cache/cache.service'; @@ -8,7 +8,9 @@ import { CacheService } from '@/services/cache/cache.service'; const GRANT_TOKEN_TTL = 15 * Time.seconds.toMilliseconds; @Service() -export class TaskRunnerAuthService { +export class TaskBrokerAuthService { + private readonly authToken = Buffer.from(this.globalConfig.taskRunners.authToken); + constructor( private readonly globalConfig: GlobalConfig, private readonly cacheService: CacheService, @@ -17,7 +19,10 @@ export class TaskRunnerAuthService { ) {} isValidAuthToken(token: string) { - return token === this.globalConfig.taskRunners.authToken; + const tokenBuffer = Buffer.from(token); + if (tokenBuffer.length !== this.authToken.length) return false; + + return timingSafeEqual(tokenBuffer, this.authToken); } /** diff --git a/packages/cli/src/task-runners/task-broker/errors/task-deferred.error.ts b/packages/cli/src/task-runners/task-broker/errors/task-deferred.error.ts new file mode 100644 index 0000000000..34f365af5f --- /dev/null +++ b/packages/cli/src/task-runners/task-broker/errors/task-deferred.error.ts @@ -0,0 +1,7 @@ +import { ApplicationError } from 'n8n-workflow'; + +export class TaskDeferredError extends ApplicationError { + constructor() { + super('Task deferred until runner is ready', { level: 'info' }); + } +} diff --git a/packages/cli/src/task-runners/errors.ts b/packages/cli/src/task-runners/task-broker/errors/task-reject.error.ts similarity index 52% rename from packages/cli/src/task-runners/errors.ts rename to packages/cli/src/task-runners/task-broker/errors/task-reject.error.ts index c530e5a95d..4e3f903dd8 100644 --- a/packages/cli/src/task-runners/errors.ts +++ b/packages/cli/src/task-runners/task-broker/errors/task-reject.error.ts @@ -5,11 +5,3 @@ export class TaskRejectError extends ApplicationError { super(`Task rejected with reason: ${reason}`, { level: 'info' }); } } - -export class TaskDeferredError extends ApplicationError { - constructor() { - super('Task deferred until runner is ready', { level: 'info' }); - } -} - -export class TaskError extends ApplicationError {} diff --git a/packages/cli/src/task-runners/errors/task-runner-timeout.error.ts b/packages/cli/src/task-runners/task-broker/errors/task-runner-timeout.error.ts similarity index 100% rename from packages/cli/src/task-runners/errors/task-runner-timeout.error.ts rename to packages/cli/src/task-runners/task-broker/errors/task-runner-timeout.error.ts diff --git a/packages/cli/src/task-runners/task-runner-server.ts b/packages/cli/src/task-runners/task-broker/task-broker-server.ts similarity index 83% rename from packages/cli/src/task-runners/task-runner-server.ts rename to packages/cli/src/task-runners/task-broker/task-broker-server.ts index 80679f8c41..ef2f0a5b28 100644 --- a/packages/cli/src/task-runners/task-runner-server.ts +++ b/packages/cli/src/task-runners/task-broker/task-broker-server.ts @@ -14,18 +14,18 @@ import { Server as WSServer } from 'ws'; import { inTest } from '@/constants'; import { bodyParser, rawBodyReader } from '@/middlewares'; import { send } from '@/response-helper'; -import { TaskRunnerAuthController } from '@/task-runners/auth/task-runner-auth.controller'; +import { TaskBrokerAuthController } from '@/task-runners/task-broker/auth/task-broker-auth.controller'; import type { - TaskRunnerServerInitRequest, - TaskRunnerServerInitResponse, -} from '@/task-runners/task-runner-types'; -import { TaskRunnerWsServer } from '@/task-runners/task-runner-ws-server'; + TaskBrokerServerInitRequest, + TaskBrokerServerInitResponse, +} from '@/task-runners/task-broker/task-broker-types'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; /** - * Task Runner HTTP & WS server + * Task Broker HTTP & WS server */ @Service() -export class TaskRunnerServer { +export class TaskBrokerServer { private server: Server | undefined; private wsServer: WSServer | undefined; @@ -43,8 +43,8 @@ export class TaskRunnerServer { constructor( private readonly logger: Logger, private readonly globalConfig: GlobalConfig, - private readonly taskRunnerAuthController: TaskRunnerAuthController, - private readonly taskRunnerWsServer: TaskRunnerWsServer, + private readonly authController: TaskBrokerAuthController, + private readonly taskBrokerWsServer: TaskBrokerWsServer, ) { this.app = express(); this.app.disable('x-powered-by'); @@ -82,7 +82,7 @@ export class TaskRunnerServer { } })(); - const stopWsServerTask = this.taskRunnerWsServer.stop(); + const stopWsServerTask = this.taskBrokerWsServer.stop(); await Promise.all([stopHttpServerTask, stopWsServerTask]); } @@ -100,7 +100,7 @@ export class TaskRunnerServer { this.server.on('error', (error: Error & { code: string }) => { if (error.code === 'EADDRINUSE') { this.logger.info( - `n8n Task Runner's port ${port} is already in use. Do you have another instance of n8n running already?`, + `n8n Task Broker's port ${port} is already in use. Do you have another instance of n8n running already?`, ); process.exit(1); } @@ -111,7 +111,7 @@ export class TaskRunnerServer { this.server.listen(port, address, () => resolve()); }); - this.logger.info(`n8n Task Runner server ready on ${address}, port ${port}`); + this.logger.info(`n8n Task Broker ready on ${address}, port ${port}`); } /** Creates WebSocket server for handling upgrade requests */ @@ -126,7 +126,7 @@ export class TaskRunnerServer { }); this.server.on('upgrade', this.handleUpgradeRequest); - this.taskRunnerWsServer.start(); + this.taskBrokerWsServer.start(); } private async setupErrorHandlers() { @@ -159,23 +159,23 @@ export class TaskRunnerServer { this.upgradeEndpoint, createRateLimiter(), // eslint-disable-next-line @typescript-eslint/unbound-method - this.taskRunnerAuthController.authMiddleware, - (req: TaskRunnerServerInitRequest, res: TaskRunnerServerInitResponse) => - this.taskRunnerWsServer.handleRequest(req, res), + this.authController.authMiddleware, + (req: TaskBrokerServerInitRequest, res: TaskBrokerServerInitResponse) => + this.taskBrokerWsServer.handleRequest(req, res), ); const authEndpoint = `${this.getEndpointBasePath()}/auth`; this.app.post( authEndpoint, createRateLimiter(), - send(async (req) => await this.taskRunnerAuthController.createGrantToken(req)), + send(async (req) => await this.authController.createGrantToken(req)), ); this.app.get('/healthz', (_, res) => res.send({ status: 'ok' })); } private handleUpgradeRequest = ( - request: TaskRunnerServerInitRequest, + request: TaskBrokerServerInitRequest, socket: Socket, head: Buffer, ) => { diff --git a/packages/cli/src/task-runners/task-runner-types.ts b/packages/cli/src/task-runners/task-broker/task-broker-types.ts similarity index 53% rename from packages/cli/src/task-runners/task-runner-types.ts rename to packages/cli/src/task-runners/task-broker/task-broker-types.ts index ca26ed41b8..71a55cc053 100644 --- a/packages/cli/src/task-runners/task-runner-types.ts +++ b/packages/cli/src/task-runners/task-broker/task-broker-types.ts @@ -1,9 +1,8 @@ +import type { TaskRunner } from '@n8n/task-runner'; import type { Response } from 'express'; -import type { INodeExecutionData } from 'n8n-workflow'; import type WebSocket from 'ws'; -import type { TaskRunner } from './task-broker.service'; -import type { AuthlessRequest } from '../requests'; +import type { AuthlessRequest } from '../../requests'; export interface DisconnectAnalyzer { isCloudDeployment: boolean; @@ -11,19 +10,12 @@ export interface DisconnectAnalyzer { toDisconnectError(opts: DisconnectErrorOptions): Promise; } -export type DataRequestType = 'input' | 'node' | 'all'; - -export interface TaskResultData { - result: INodeExecutionData[]; - customData?: Record; -} - -export interface TaskRunnerServerInitRequest +export interface TaskBrokerServerInitRequest extends AuthlessRequest<{}, {}, {}, { id: TaskRunner['id']; token?: string }> { ws: WebSocket; } -export type TaskRunnerServerInitResponse = Response & { req: TaskRunnerServerInitRequest }; +export type TaskBrokerServerInitResponse = Response & { req: TaskBrokerServerInitRequest }; export type DisconnectReason = 'shutting-down' | 'failed-heartbeat-check' | 'unknown'; diff --git a/packages/cli/src/task-runners/task-runner-ws-server.ts b/packages/cli/src/task-runners/task-broker/task-broker-ws-server.ts similarity index 90% rename from packages/cli/src/task-runners/task-runner-ws-server.ts rename to packages/cli/src/task-runners/task-broker/task-broker-ws-server.ts index dbec7cef3e..7b65140717 100644 --- a/packages/cli/src/task-runners/task-runner-ws-server.ts +++ b/packages/cli/src/task-runners/task-broker/task-broker-ws-server.ts @@ -6,16 +6,16 @@ import { ApplicationError, jsonStringify } from 'n8n-workflow'; import type WebSocket from 'ws'; import { Time, WsStatusCodes } from '@/constants'; - -import { DefaultTaskRunnerDisconnectAnalyzer } from './default-task-runner-disconnect-analyzer'; -import { TaskBroker, type MessageCallback, type TaskRunner } from './task-broker.service'; -import { TaskRunnerLifecycleEvents } from './task-runner-lifecycle-events'; +import { DefaultTaskRunnerDisconnectAnalyzer } from '@/task-runners/default-task-runner-disconnect-analyzer'; import type { DisconnectAnalyzer, DisconnectReason, - TaskRunnerServerInitRequest, - TaskRunnerServerInitResponse, -} from './task-runner-types'; + TaskBrokerServerInitRequest, + TaskBrokerServerInitResponse, +} from '@/task-runners/task-broker/task-broker-types'; +import { TaskRunnerLifecycleEvents } from '@/task-runners/task-runner-lifecycle-events'; + +import { TaskBroker, type MessageCallback, type TaskRunner } from './task-broker.service'; function heartbeat(this: WebSocket) { this.isAlive = true; @@ -23,8 +23,12 @@ function heartbeat(this: WebSocket) { type WsStatusCode = (typeof WsStatusCodes)[keyof typeof WsStatusCodes]; +/** + * Responsible for handling WebSocket connections with task runners + * and monitoring the connection liveness + */ @Service() -export class TaskRunnerWsServer { +export class TaskBrokerWsServer { runnerConnections: Map = new Map(); private heartbeatTimer: NodeJS.Timer | undefined; @@ -164,7 +168,7 @@ export class TaskRunnerWsServer { } } - handleRequest(req: TaskRunnerServerInitRequest, _res: TaskRunnerServerInitResponse) { + handleRequest(req: TaskBrokerServerInitRequest, _res: TaskBrokerServerInitResponse) { this.add(req.query.id, req.ws); } diff --git a/packages/cli/src/task-runners/task-broker.service.ts b/packages/cli/src/task-runners/task-broker/task-broker.service.ts similarity index 97% rename from packages/cli/src/task-runners/task-broker.service.ts rename to packages/cli/src/task-runners/task-broker/task-broker.service.ts index 42e0d3cd25..3afd182914 100644 --- a/packages/cli/src/task-runners/task-broker.service.ts +++ b/packages/cli/src/task-runners/task-broker/task-broker.service.ts @@ -12,10 +12,10 @@ import { nanoid } from 'nanoid'; import config from '@/config'; import { Time } from '@/constants'; - -import { TaskDeferredError, TaskRejectError } from './errors'; -import { TaskRunnerTimeoutError } from './errors/task-runner-timeout.error'; -import { TaskRunnerLifecycleEvents } from './task-runner-lifecycle-events'; +import { TaskDeferredError } from '@/task-runners/task-broker/errors/task-deferred.error'; +import { TaskRejectError } from '@/task-runners/task-broker/errors/task-reject.error'; +import { TaskRunnerTimeoutError } from '@/task-runners/task-broker/errors/task-runner-timeout.error'; +import { TaskRunnerLifecycleEvents } from '@/task-runners/task-runner-lifecycle-events'; export interface TaskRunner { id: string; diff --git a/packages/cli/src/task-runners/task-managers/local-task-requester.ts b/packages/cli/src/task-runners/task-managers/local-task-requester.ts index 1de959936b..4fc45ccd6c 100644 --- a/packages/cli/src/task-runners/task-managers/local-task-requester.ts +++ b/packages/cli/src/task-runners/task-managers/local-task-requester.ts @@ -2,10 +2,10 @@ import { Container, Service } from '@n8n/di'; import type { RequesterMessage } from '@n8n/task-runner'; import { NodeTypes } from '@/node-types'; +import type { RequesterMessageCallback } from '@/task-runners/task-broker/task-broker.service'; +import { TaskBroker } from '@/task-runners/task-broker/task-broker.service'; import { TaskRequester } from './task-requester'; -import type { RequesterMessageCallback } from '../task-broker.service'; -import { TaskBroker } from '../task-broker.service'; @Service() export class LocalTaskRequester extends TaskRequester { diff --git a/packages/cli/src/task-runners/task-runner-module.ts b/packages/cli/src/task-runners/task-runner-module.ts index bcc54f653f..0ba60123f9 100644 --- a/packages/cli/src/task-runners/task-runner-module.ts +++ b/packages/cli/src/task-runners/task-runner-module.ts @@ -6,13 +6,13 @@ import * as a from 'node:assert/strict'; import { OnShutdown } from '@/decorators/on-shutdown'; import type { TaskRunnerRestartLoopError } from '@/task-runners/errors/task-runner-restart-loop-error'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; import type { TaskRunnerProcess } from '@/task-runners/task-runner-process'; import { TaskRunnerProcessRestartLoopDetector } from '@/task-runners/task-runner-process-restart-loop-detector'; import { MissingAuthTokenError } from './errors/missing-auth-token.error'; +import type { TaskBrokerServer } from './task-broker/task-broker-server'; import type { LocalTaskRequester } from './task-managers/local-task-requester'; -import type { TaskRunnerServer } from './task-runner-server'; -import { TaskRunnerWsServer } from './task-runner-ws-server'; /** * Module responsible for loading and starting task runner. Task runner can be @@ -21,9 +21,9 @@ import { TaskRunnerWsServer } from './task-runner-ws-server'; */ @Service() export class TaskRunnerModule { - private taskRunnerHttpServer: TaskRunnerServer | undefined; + private taskBrokerHttpServer: TaskBrokerServer | undefined; - private taskRunnerWsServer: TaskRunnerWsServer | undefined; + private taskBrokerWsServer: TaskBrokerWsServer | undefined; private taskRequester: LocalTaskRequester | undefined; @@ -47,7 +47,7 @@ export class TaskRunnerModule { if (mode === 'external' && !authToken) throw new MissingAuthTokenError(); await this.loadTaskRequester(); - await this.loadTaskRunnerServer(); + await this.loadTaskBroker(); if (mode === 'internal') { await this.startInternalTaskRunner(); @@ -64,9 +64,9 @@ export class TaskRunnerModule { })(); const stopRunnerServerTask = (async () => { - if (this.taskRunnerHttpServer) { - await this.taskRunnerHttpServer.stop(); - this.taskRunnerHttpServer = undefined; + if (this.taskBrokerHttpServer) { + await this.taskBrokerHttpServer.stop(); + this.taskBrokerHttpServer = undefined; } })(); @@ -82,18 +82,18 @@ export class TaskRunnerModule { Container.set(TaskRequester, this.taskRequester); } - private async loadTaskRunnerServer() { + private async loadTaskBroker() { // These are imported dynamically because we need to set the task manager // instance before importing them - const { TaskRunnerServer } = await import('@/task-runners/task-runner-server'); - this.taskRunnerHttpServer = Container.get(TaskRunnerServer); - this.taskRunnerWsServer = Container.get(TaskRunnerWsServer); + const { TaskBrokerServer } = await import('@/task-runners/task-broker/task-broker-server'); + this.taskBrokerHttpServer = Container.get(TaskBrokerServer); + this.taskBrokerWsServer = Container.get(TaskBrokerWsServer); - await this.taskRunnerHttpServer.start(); + await this.taskBrokerHttpServer.start(); } private async startInternalTaskRunner() { - a.ok(this.taskRunnerWsServer, 'Task Runner WS Server not loaded'); + a.ok(this.taskBrokerWsServer, 'Task Runner WS Server not loaded'); const { TaskRunnerProcess } = await import('@/task-runners/task-runner-process'); this.taskRunnerProcess = Container.get(TaskRunnerProcess); @@ -110,7 +110,7 @@ export class TaskRunnerModule { const { InternalTaskRunnerDisconnectAnalyzer } = await import( '@/task-runners/internal-task-runner-disconnect-analyzer' ); - this.taskRunnerWsServer.setDisconnectAnalyzer( + this.taskBrokerWsServer.setDisconnectAnalyzer( Container.get(InternalTaskRunnerDisconnectAnalyzer), ); } diff --git a/packages/cli/src/task-runners/task-runner-process.ts b/packages/cli/src/task-runners/task-runner-process.ts index 5129ae98b1..26cfefc469 100644 --- a/packages/cli/src/task-runners/task-runner-process.ts +++ b/packages/cli/src/task-runners/task-runner-process.ts @@ -7,9 +7,9 @@ import * as process from 'node:process'; import { OnShutdown } from '@/decorators/on-shutdown'; -import { TaskRunnerAuthService } from './auth/task-runner-auth.service'; import { forwardToLogger } from './forward-to-logger'; import { NodeProcessOomDetector } from './node-process-oom-detector'; +import { TaskBrokerAuthService } from './task-broker/auth/task-broker-auth.service'; import { TaskRunnerLifecycleEvents } from './task-runner-lifecycle-events'; import { TypedEmitter } from '../typed-emitter'; @@ -54,6 +54,7 @@ export class TaskRunnerProcess extends TypedEmitter { private readonly passthroughEnvVars = [ 'PATH', + 'HOME', // So home directory can be resolved correctly 'GENERIC_TIMEZONE', 'NODE_FUNCTION_ALLOW_BUILTIN', 'NODE_FUNCTION_ALLOW_EXTERNAL', @@ -67,7 +68,7 @@ export class TaskRunnerProcess extends TypedEmitter { constructor( logger: Logger, private readonly runnerConfig: TaskRunnersConfig, - private readonly authService: TaskRunnerAuthService, + private readonly authService: TaskBrokerAuthService, private readonly runnerLifecycleEvents: TaskRunnerLifecycleEvents, ) { super(); @@ -106,9 +107,13 @@ export class TaskRunnerProcess extends TypedEmitter { startNode(grantToken: string, taskBrokerUri: string) { const startScript = require.resolve('@n8n/task-runner/start'); - return spawn('node', [startScript], { - env: this.getProcessEnvVars(grantToken, taskBrokerUri), - }); + return spawn( + 'node', + ['--disallow-code-generation-from-strings', '--disable-proto=delete', startScript], + { + env: this.getProcessEnvVars(grantToken, taskBrokerUri), + }, + ); } @OnShutdown() diff --git a/packages/cli/src/__tests__/object-to-error.test.ts b/packages/cli/src/utils/__tests__/object-to-error.test.ts similarity index 94% rename from packages/cli/src/__tests__/object-to-error.test.ts rename to packages/cli/src/utils/__tests__/object-to-error.test.ts index 311f4dce55..c65676a426 100644 --- a/packages/cli/src/__tests__/object-to-error.test.ts +++ b/packages/cli/src/utils/__tests__/object-to-error.test.ts @@ -2,7 +2,7 @@ import { mock } from 'jest-mock-extended'; import type { INode } from 'n8n-workflow'; import { NodeOperationError, type Workflow } from 'n8n-workflow'; -import { objectToError } from '../workflow-execute-additional-data'; +import { objectToError } from '../object-to-error'; describe('objectToError', () => { describe('node error handling', () => { diff --git a/packages/cli/src/utils/object-to-error.ts b/packages/cli/src/utils/object-to-error.ts new file mode 100644 index 0000000000..ffb0cd8fb3 --- /dev/null +++ b/packages/cli/src/utils/object-to-error.ts @@ -0,0 +1,53 @@ +import { isObjectLiteral } from 'n8n-core'; +import { NodeOperationError } from 'n8n-workflow'; +import type { Workflow } from 'n8n-workflow'; + +export function objectToError(errorObject: unknown, workflow: Workflow): Error { + // TODO: Expand with other error types + if (errorObject instanceof Error) { + // If it's already an Error instance, return it as is. + return errorObject; + } else if ( + isObjectLiteral(errorObject) && + 'message' in errorObject && + typeof errorObject.message === 'string' + ) { + // If it's an object with a 'message' property, create a new Error instance. + let error: Error | undefined; + if ( + 'node' in errorObject && + isObjectLiteral(errorObject.node) && + typeof errorObject.node.name === 'string' + ) { + const node = workflow.getNode(errorObject.node.name); + + if (node) { + error = new NodeOperationError( + node, + errorObject as unknown as Error, + errorObject as object, + ); + } + } + + if (error === undefined) { + error = new Error(errorObject.message); + } + + if ('description' in errorObject) { + // @ts-expect-error Error descriptions are surfaced by the UI but + // not all backend errors account for this property yet. + error.description = errorObject.description as string; + } + + if ('stack' in errorObject) { + // If there's a 'stack' property, set it on the new Error instance. + error.stack = errorObject.stack as string; + } + + return error; + } else { + // If it's neither an Error nor an object with a 'message' property, create a generic Error. + return new Error('An error occurred'); + } +} diff --git a/packages/cli/src/webhooks/webhook-helpers.ts b/packages/cli/src/webhooks/webhook-helpers.ts index 1711d18056..ca54a14b52 100644 --- a/packages/cli/src/webhooks/webhook-helpers.ts +++ b/packages/cli/src/webhooks/webhook-helpers.ts @@ -454,7 +454,7 @@ export async function executeWebhook( } let pinData: IPinData | undefined; - const usePinData = executionMode === 'manual'; + const usePinData = ['manual', 'evaluation'].includes(executionMode); if (usePinData) { pinData = workflowData.pinData; runExecutionData.resultData.pinData = pinData; @@ -530,6 +530,7 @@ export async function executeWebhook( `Error with Webhook-Response for execution "${executionId}": "${error.message}"`, { executionId, workflowId: workflow.id }, ); + responseCallback(error, {}); }); } diff --git a/packages/cli/src/webhooks/webhook-request-handler.ts b/packages/cli/src/webhooks/webhook-request-handler.ts index 96cf6da745..8eb62358b1 100644 --- a/packages/cli/src/webhooks/webhook-request-handler.ts +++ b/packages/cli/src/webhooks/webhook-request-handler.ts @@ -1,5 +1,7 @@ +import { Container } from '@n8n/di'; import type express from 'express'; -import type { IHttpRequestMethods } from 'n8n-workflow'; +import { Logger } from 'n8n-core'; +import { ensureError, type IHttpRequestMethods } from 'n8n-workflow'; import * as ResponseHelper from '@/response-helper'; import type { @@ -52,8 +54,14 @@ class WebhookRequestHandler { response.headers, ); } - } catch (error) { - return ResponseHelper.sendErrorResponse(res, error as Error); + } catch (e) { + const error = ensureError(e); + Container.get(Logger).debug( + `Error in handling webhook request ${req.method} ${req.path}: ${error.message}`, + { stacktrace: error.stack }, + ); + + return ResponseHelper.sendErrorResponse(res, error); } } diff --git a/packages/cli/src/workflow-execute-additional-data.ts b/packages/cli/src/workflow-execute-additional-data.ts index e350086f9d..1c5dfcbf67 100644 --- a/packages/cli/src/workflow-execute-additional-data.ts +++ b/packages/cli/src/workflow-execute-additional-data.ts @@ -5,15 +5,8 @@ import type { PushMessage, PushType } from '@n8n/api-types'; import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; -import { stringify } from 'flatted'; -import { - ErrorReporter, - Logger, - InstanceSettings, - WorkflowExecute, - isObjectLiteral, -} from 'n8n-core'; -import { ApplicationError, NodeOperationError, Workflow, WorkflowHooks } from 'n8n-workflow'; +import { Logger, WorkflowExecute } from 'n8n-core'; +import { ApplicationError, Workflow } from 'n8n-workflow'; import type { IDataObject, IExecuteData, @@ -23,11 +16,8 @@ import type { INodeParameters, IRun, IRunExecutionData, - ITaskData, IWorkflowBase, IWorkflowExecuteAdditionalData, - IWorkflowExecuteHooks, - IWorkflowHooksOptionalParameters, IWorkflowSettings, WorkflowExecuteMode, ExecutionStatus, @@ -42,636 +32,24 @@ import type { } from 'n8n-workflow'; import { ActiveExecutions } from '@/active-executions'; -import config from '@/config'; import { CredentialsHelper } from '@/credentials-helper'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import { EventService } from '@/events/event.service'; import type { AiEventMap, AiEventPayload } from '@/events/maps/ai.event-map'; -import { ExternalHooks } from '@/external-hooks'; -import type { IWorkflowErrorData, UpdateExecutionPayload } from '@/interfaces'; +import { getLifecycleHooksForSubExecutions } from '@/execution-lifecycle/execution-lifecycle-hooks'; +import type { UpdateExecutionPayload } from '@/interfaces'; import { NodeTypes } from '@/node-types'; import { Push } from '@/push'; -import { WorkflowStatisticsService } from '@/services/workflow-statistics.service'; -import { findSubworkflowStart, isWorkflowIdValid } from '@/utils'; +import { SecretsHelper } from '@/secrets-helpers.ee'; +import { UrlService } from '@/services/url.service'; +import { SubworkflowPolicyChecker } from '@/subworkflows/subworkflow-policy-checker.service'; +import { TaskRequester } from '@/task-runners/task-managers/task-requester'; +import { PermissionChecker } from '@/user-management/permission-checker'; +import { findSubworkflowStart } from '@/utils'; +import { objectToError } from '@/utils/object-to-error'; import * as WorkflowHelpers from '@/workflow-helpers'; -import { WorkflowRepository } from './databases/repositories/workflow.repository'; -import { EventService } from './events/event.service'; -import { restoreBinaryDataId } from './execution-lifecycle-hooks/restore-binary-data-id'; -import { saveExecutionProgress } from './execution-lifecycle-hooks/save-execution-progress'; -import { - determineFinalExecutionStatus, - prepareExecutionDataForDbUpdate, - updateExistingExecution, -} from './execution-lifecycle-hooks/shared/shared-hook-functions'; -import { toSaveSettings } from './execution-lifecycle-hooks/to-save-settings'; -import { SecretsHelper } from './secrets-helpers.ee'; -import { OwnershipService } from './services/ownership.service'; -import { UrlService } from './services/url.service'; -import { SubworkflowPolicyChecker } from './subworkflows/subworkflow-policy-checker.service'; -import { TaskRequester } from './task-runners/task-managers/task-requester'; -import { PermissionChecker } from './user-management/permission-checker'; -import { WorkflowExecutionService } from './workflows/workflow-execution.service'; -import { WorkflowStaticDataService } from './workflows/workflow-static-data.service'; - -export function objectToError(errorObject: unknown, workflow: Workflow): Error { - // TODO: Expand with other error types - if (errorObject instanceof Error) { - // If it's already an Error instance, return it as is. - return errorObject; - } else if ( - isObjectLiteral(errorObject) && - 'message' in errorObject && - typeof errorObject.message === 'string' - ) { - // If it's an object with a 'message' property, create a new Error instance. - let error: Error | undefined; - if ( - 'node' in errorObject && - isObjectLiteral(errorObject.node) && - typeof errorObject.node.name === 'string' - ) { - const node = workflow.getNode(errorObject.node.name); - - if (node) { - error = new NodeOperationError( - node, - errorObject as unknown as Error, - errorObject as object, - ); - } - } - - if (error === undefined) { - error = new Error(errorObject.message); - } - - if ('description' in errorObject) { - // @ts-expect-error Error descriptions are surfaced by the UI but - // not all backend errors account for this property yet. - error.description = errorObject.description as string; - } - - if ('stack' in errorObject) { - // If there's a 'stack' property, set it on the new Error instance. - error.stack = errorObject.stack as string; - } - - return error; - } else { - // If it's neither an Error nor an object with a 'message' property, create a generic Error. - return new Error('An error occurred'); - } -} - -/** - * Checks if there was an error and if errorWorkflow or a trigger is defined. If so it collects - * all the data and executes it - * - * @param {IWorkflowBase} workflowData The workflow which got executed - * @param {IRun} fullRunData The run which produced the error - * @param {WorkflowExecuteMode} mode The mode in which the workflow got started in - * @param {string} [executionId] The id the execution got saved as - */ -export function executeErrorWorkflow( - workflowData: IWorkflowBase, - fullRunData: IRun, - mode: WorkflowExecuteMode, - executionId?: string, - retryOf?: string, -): void { - const logger = Container.get(Logger); - - // Check if there was an error and if so if an errorWorkflow or a trigger is set - let pastExecutionUrl: string | undefined; - if (executionId !== undefined) { - pastExecutionUrl = `${Container.get(UrlService).getWebhookBaseUrl()}workflow/${ - workflowData.id - }/executions/${executionId}`; - } - - if (fullRunData.data.resultData.error !== undefined) { - let workflowErrorData: IWorkflowErrorData; - const workflowId = workflowData.id; - - if (executionId) { - // The error did happen in an execution - workflowErrorData = { - execution: { - id: executionId, - url: pastExecutionUrl, - error: fullRunData.data.resultData.error, - lastNodeExecuted: fullRunData.data.resultData.lastNodeExecuted!, - mode, - retryOf, - }, - workflow: { - id: workflowId, - name: workflowData.name, - }, - }; - } else { - // The error did happen in a trigger - workflowErrorData = { - trigger: { - error: fullRunData.data.resultData.error, - mode, - }, - workflow: { - id: workflowId, - name: workflowData.name, - }, - }; - } - - const { errorTriggerType } = Container.get(GlobalConfig).nodes; - // Run the error workflow - // To avoid an infinite loop do not run the error workflow again if the error-workflow itself failed and it is its own error-workflow. - const { errorWorkflow } = workflowData.settings ?? {}; - if (errorWorkflow && !(mode === 'error' && workflowId && errorWorkflow === workflowId)) { - logger.debug('Start external error workflow', { - executionId, - errorWorkflowId: errorWorkflow, - workflowId, - }); - // If a specific error workflow is set run only that one - - // First, do permission checks. - if (!workflowId) { - // Manual executions do not trigger error workflows - // So this if should never happen. It was added to - // make sure there are no possible security gaps - return; - } - - Container.get(OwnershipService) - .getWorkflowProjectCached(workflowId) - .then((project) => { - void Container.get(WorkflowExecutionService).executeErrorWorkflow( - errorWorkflow, - workflowErrorData, - project, - ); - }) - .catch((error: Error) => { - Container.get(ErrorReporter).error(error); - logger.error( - `Could not execute ErrorWorkflow for execution ID ${this.executionId} because of error querying the workflow owner`, - { - executionId, - errorWorkflowId: errorWorkflow, - workflowId, - error, - workflowErrorData, - }, - ); - }); - } else if ( - mode !== 'error' && - workflowId !== undefined && - workflowData.nodes.some((node) => node.type === errorTriggerType) - ) { - logger.debug('Start internal error workflow', { executionId, workflowId }); - void Container.get(OwnershipService) - .getWorkflowProjectCached(workflowId) - .then((project) => { - void Container.get(WorkflowExecutionService).executeErrorWorkflow( - workflowId, - workflowErrorData, - project, - ); - }); - } - } -} - -/** - * Returns hook functions to push data to Editor-UI - * - */ -function hookFunctionsPush(): IWorkflowExecuteHooks { - const logger = Container.get(Logger); - const pushInstance = Container.get(Push); - return { - nodeExecuteBefore: [ - async function (this: WorkflowHooks, nodeName: string): Promise { - const { pushRef, executionId } = this; - // Push data to session which started workflow before each - // node which starts rendering - if (pushRef === undefined) { - return; - } - - logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { - executionId, - pushRef, - workflowId: this.workflowData.id, - }); - - pushInstance.send({ type: 'nodeExecuteBefore', data: { executionId, nodeName } }, pushRef); - }, - ], - nodeExecuteAfter: [ - async function (this: WorkflowHooks, nodeName: string, data: ITaskData): Promise { - const { pushRef, executionId } = this; - // Push data to session which started workflow after each rendered node - if (pushRef === undefined) { - return; - } - - logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { - executionId, - pushRef, - workflowId: this.workflowData.id, - }); - - pushInstance.send( - { type: 'nodeExecuteAfter', data: { executionId, nodeName, data } }, - pushRef, - ); - }, - ], - workflowExecuteBefore: [ - async function (this: WorkflowHooks, _workflow, data): Promise { - const { pushRef, executionId } = this; - const { id: workflowId, name: workflowName } = this.workflowData; - logger.debug('Executing hook (hookFunctionsPush)', { - executionId, - pushRef, - workflowId, - }); - // Push data to session which started the workflow - if (pushRef === undefined) { - return; - } - pushInstance.send( - { - type: 'executionStarted', - data: { - executionId, - mode: this.mode, - startedAt: new Date(), - retryOf: this.retryOf, - workflowId, - workflowName, - flattedRunData: data?.resultData.runData - ? stringify(data.resultData.runData) - : stringify({}), - }, - }, - pushRef, - ); - }, - ], - workflowExecuteAfter: [ - async function (this: WorkflowHooks, fullRunData: IRun): Promise { - const { pushRef, executionId } = this; - if (pushRef === undefined) return; - - const { id: workflowId } = this.workflowData; - logger.debug('Executing hook (hookFunctionsPush)', { - executionId, - pushRef, - workflowId, - }); - - const { status } = fullRunData; - if (status === 'waiting') { - pushInstance.send({ type: 'executionWaiting', data: { executionId } }, pushRef); - } else { - const rawData = stringify(fullRunData.data); - pushInstance.send( - { type: 'executionFinished', data: { executionId, workflowId, status, rawData } }, - pushRef, - ); - } - }, - ], - }; -} - -export function hookFunctionsPreExecute(): IWorkflowExecuteHooks { - const externalHooks = Container.get(ExternalHooks); - return { - workflowExecuteBefore: [ - async function (this: WorkflowHooks, workflow: Workflow): Promise { - await externalHooks.run('workflow.preExecute', [workflow, this.mode]); - }, - ], - nodeExecuteAfter: [ - async function ( - this: WorkflowHooks, - nodeName: string, - data: ITaskData, - executionData: IRunExecutionData, - ): Promise { - await saveExecutionProgress( - this.workflowData, - this.executionId, - nodeName, - data, - executionData, - this.pushRef, - ); - }, - ], - }; -} - -/** - * Returns hook functions to save workflow execution and call error workflow - * - */ -function hookFunctionsSave(): IWorkflowExecuteHooks { - const logger = Container.get(Logger); - const workflowStatisticsService = Container.get(WorkflowStatisticsService); - const eventService = Container.get(EventService); - return { - nodeExecuteBefore: [ - async function (this: WorkflowHooks, nodeName: string): Promise { - const { executionId, workflowData: workflow } = this; - - eventService.emit('node-pre-execute', { executionId, workflow, nodeName }); - }, - ], - nodeExecuteAfter: [ - async function (this: WorkflowHooks, nodeName: string): Promise { - const { executionId, workflowData: workflow } = this; - - eventService.emit('node-post-execute', { executionId, workflow, nodeName }); - }, - ], - workflowExecuteBefore: [], - workflowExecuteAfter: [ - async function ( - this: WorkflowHooks, - fullRunData: IRun, - newStaticData: IDataObject, - ): Promise { - logger.debug('Executing hook (hookFunctionsSave)', { - executionId: this.executionId, - workflowId: this.workflowData.id, - }); - - await restoreBinaryDataId(fullRunData, this.executionId, this.mode); - - const isManualMode = this.mode === 'manual'; - - try { - if (!isManualMode && isWorkflowIdValid(this.workflowData.id) && newStaticData) { - // Workflow is saved so update in database - try { - await Container.get(WorkflowStaticDataService).saveStaticDataById( - this.workflowData.id, - newStaticData, - ); - } catch (e) { - Container.get(ErrorReporter).error(e); - logger.error( - `There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (hookFunctionsSave)`, - { executionId: this.executionId, workflowId: this.workflowData.id }, - ); - } - } - - const executionStatus = determineFinalExecutionStatus(fullRunData); - fullRunData.status = executionStatus; - - const saveSettings = toSaveSettings(this.workflowData.settings); - - if (isManualMode && !saveSettings.manual && !fullRunData.waitTill) { - /** - * When manual executions are not being saved, we only soft-delete - * the execution so that the user can access its binary data - * while building their workflow. - * - * The manual execution and its binary data will be hard-deleted - * on the next pruning cycle after the grace period set by - * `EXECUTIONS_DATA_HARD_DELETE_BUFFER`. - */ - await Container.get(ExecutionRepository).softDelete(this.executionId); - - return; - } - - const shouldNotSave = - (executionStatus === 'success' && !saveSettings.success) || - (executionStatus !== 'success' && !saveSettings.error); - - if (shouldNotSave && !fullRunData.waitTill && !isManualMode) { - executeErrorWorkflow( - this.workflowData, - fullRunData, - this.mode, - this.executionId, - this.retryOf, - ); - - await Container.get(ExecutionRepository).hardDelete({ - workflowId: this.workflowData.id, - executionId: this.executionId, - }); - - return; - } - - // Although it is treated as IWorkflowBase here, it's being instantiated elsewhere with properties that may be sensitive - // As a result, we should create an IWorkflowBase object with only the data we want to save in it. - const fullExecutionData = prepareExecutionDataForDbUpdate({ - runData: fullRunData, - workflowData: this.workflowData, - workflowStatusFinal: executionStatus, - retryOf: this.retryOf, - }); - - // When going into the waiting state, store the pushRef in the execution-data - if (fullRunData.waitTill && isManualMode) { - fullExecutionData.data.pushRef = this.pushRef; - } - - await updateExistingExecution({ - executionId: this.executionId, - workflowId: this.workflowData.id, - executionData: fullExecutionData, - }); - - if (!isManualMode) { - executeErrorWorkflow( - this.workflowData, - fullRunData, - this.mode, - this.executionId, - this.retryOf, - ); - } - } catch (error) { - Container.get(ErrorReporter).error(error); - logger.error(`Failed saving execution data to DB on execution ID ${this.executionId}`, { - executionId: this.executionId, - workflowId: this.workflowData.id, - error, - }); - if (!isManualMode) { - executeErrorWorkflow( - this.workflowData, - fullRunData, - this.mode, - this.executionId, - this.retryOf, - ); - } - } finally { - workflowStatisticsService.emit('workflowExecutionCompleted', { - workflowData: this.workflowData, - fullRunData, - }); - } - }, - ], - nodeFetchedData: [ - async (workflowId: string, node: INode) => { - workflowStatisticsService.emit('nodeFetchedData', { workflowId, node }); - }, - ], - }; -} - -/** - * Returns hook functions to save workflow execution and call error workflow - * for running with queues. Manual executions should never run on queues as - * they are always executed in the main process. - * - */ -function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { - const logger = Container.get(Logger); - const workflowStatisticsService = Container.get(WorkflowStatisticsService); - const eventService = Container.get(EventService); - return { - nodeExecuteBefore: [ - async function (this: WorkflowHooks, nodeName: string): Promise { - const { executionId, workflowData: workflow } = this; - - eventService.emit('node-pre-execute', { executionId, workflow, nodeName }); - }, - ], - nodeExecuteAfter: [ - async function (this: WorkflowHooks, nodeName: string): Promise { - const { executionId, workflowData: workflow } = this; - - eventService.emit('node-post-execute', { executionId, workflow, nodeName }); - }, - ], - workflowExecuteBefore: [ - async function (): Promise { - const { executionId, workflowData } = this; - - eventService.emit('workflow-pre-execute', { executionId, data: workflowData }); - }, - ], - workflowExecuteAfter: [ - async function ( - this: WorkflowHooks, - fullRunData: IRun, - newStaticData: IDataObject, - ): Promise { - logger.debug('Executing hook (hookFunctionsSaveWorker)', { - executionId: this.executionId, - workflowId: this.workflowData.id, - }); - try { - if (isWorkflowIdValid(this.workflowData.id) && newStaticData) { - // Workflow is saved so update in database - try { - await Container.get(WorkflowStaticDataService).saveStaticDataById( - this.workflowData.id, - newStaticData, - ); - } catch (e) { - Container.get(ErrorReporter).error(e); - logger.error( - `There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (workflowExecuteAfter)`, - { pushRef: this.pushRef, workflowId: this.workflowData.id }, - ); - } - } - - const workflowStatusFinal = determineFinalExecutionStatus(fullRunData); - fullRunData.status = workflowStatusFinal; - - if (workflowStatusFinal !== 'success' && workflowStatusFinal !== 'waiting') { - executeErrorWorkflow( - this.workflowData, - fullRunData, - this.mode, - this.executionId, - this.retryOf, - ); - } - - // Although it is treated as IWorkflowBase here, it's being instantiated elsewhere with properties that may be sensitive - // As a result, we should create an IWorkflowBase object with only the data we want to save in it. - const fullExecutionData = prepareExecutionDataForDbUpdate({ - runData: fullRunData, - workflowData: this.workflowData, - workflowStatusFinal, - retryOf: this.retryOf, - }); - - await updateExistingExecution({ - executionId: this.executionId, - workflowId: this.workflowData.id, - executionData: fullExecutionData, - }); - } catch (error) { - executeErrorWorkflow( - this.workflowData, - fullRunData, - this.mode, - this.executionId, - this.retryOf, - ); - } finally { - workflowStatisticsService.emit('workflowExecutionCompleted', { - workflowData: this.workflowData, - fullRunData, - }); - } - }, - async function (this: WorkflowHooks, runData: IRun): Promise { - const { executionId, workflowData: workflow } = this; - - eventService.emit('workflow-post-execute', { - workflow, - executionId, - runData, - }); - }, - async function (this: WorkflowHooks, fullRunData: IRun) { - const externalHooks = Container.get(ExternalHooks); - if (externalHooks.exists('workflow.postExecute')) { - try { - await externalHooks.run('workflow.postExecute', [ - fullRunData, - this.workflowData, - this.executionId, - ]); - } catch (error) { - Container.get(ErrorReporter).error(error); - Container.get(Logger).error( - 'There was a problem running hook "workflow.postExecute"', - error, - ); - } - } - }, - ], - nodeFetchedData: [ - async (workflowId: string, node: INode) => { - workflowStatisticsService.emit('nodeFetchedData', { workflowId, node }); - }, - ], - }; -} - export async function getRunData( workflowData: IWorkflowBase, inputData?: INodeExecutionData[], @@ -734,7 +112,7 @@ export async function getWorkflowData( let workflowData: IWorkflowBase | null; if (workflowInfo.id !== undefined) { - const relations = config.getEnv('workflowTagsDisabled') ? [] : ['tags']; + const relations = Container.get(GlobalConfig).tags.disabled ? [] : ['tags']; workflowData = await Container.get(WorkflowRepository).get( { id: workflowInfo.id }, @@ -803,12 +181,8 @@ async function startExecution( runData: IWorkflowExecutionDataProcess, workflowData: IWorkflowBase, ): Promise { - const externalHooks = Container.get(ExternalHooks); - await externalHooks.init(); - const nodeTypes = Container.get(NodeTypes); const activeExecutions = Container.get(ActiveExecutions); - const eventService = Container.get(EventService); const executionRepository = Container.get(ExecutionRepository); const workflowName = workflowData ? workflowData.name : undefined; @@ -830,8 +204,6 @@ async function startExecution( */ await executionRepository.setRunning(executionId); - Container.get(EventService).emit('workflow-pre-execute', { executionId, data: runData }); - let data; try { await Container.get(PermissionChecker).check(workflowData.id, workflowData.nodes); @@ -845,10 +217,11 @@ async function startExecution( // Create new additionalData to have different workflow loaded and to call // different webhooks const additionalDataIntegrated = await getBase(); - additionalDataIntegrated.hooks = getWorkflowHooksIntegrated( + additionalDataIntegrated.hooks = getLifecycleHooksForSubExecutions( runData.executionMode, executionId, workflowData, + additionalData.userId, ); additionalDataIntegrated.executionId = executionId; additionalDataIntegrated.parentCallbackManager = options.parentCallbackManager; @@ -929,15 +302,6 @@ async function startExecution( ); } - await externalHooks.run('workflow.postExecute', [data, workflowData, executionId]); - - eventService.emit('workflow-post-execute', { - workflow: workflowData, - executionId, - userId: additionalData.userId, - runData: data, - }); - // subworkflow either finished, or is in status waiting due to a wait node, both cases are considered successes here if (data.finished === true || data.status === 'waiting') { // Workflow did finish successfully @@ -1062,137 +426,3 @@ export async function getBase( eventService.emit(eventName, payload), }; } - -/** - * Returns WorkflowHooks instance for running integrated workflows - * (Workflows which get started inside of another workflow) - */ -function getWorkflowHooksIntegrated( - mode: WorkflowExecuteMode, - executionId: string, - workflowData: IWorkflowBase, -): WorkflowHooks { - const hookFunctions = hookFunctionsSave(); - const preExecuteFunctions = hookFunctionsPreExecute(); - for (const key of Object.keys(preExecuteFunctions)) { - const hooks = hookFunctions[key] ?? []; - hooks.push.apply(hookFunctions[key], preExecuteFunctions[key]); - } - return new WorkflowHooks(hookFunctions, mode, executionId, workflowData); -} - -/** - * Returns WorkflowHooks instance for worker in scaling mode. - */ -export function getWorkflowHooksWorkerExecuter( - mode: WorkflowExecuteMode, - executionId: string, - workflowData: IWorkflowBase, - optionalParameters?: IWorkflowHooksOptionalParameters, -): WorkflowHooks { - optionalParameters = optionalParameters || {}; - const hookFunctions = hookFunctionsSaveWorker(); - const preExecuteFunctions = hookFunctionsPreExecute(); - for (const key of Object.keys(preExecuteFunctions)) { - const hooks = hookFunctions[key] ?? []; - hooks.push.apply(hookFunctions[key], preExecuteFunctions[key]); - } - - if (mode === 'manual' && Container.get(InstanceSettings).isWorker) { - const pushHooks = hookFunctionsPush(); - for (const key of Object.keys(pushHooks)) { - if (hookFunctions[key] === undefined) { - hookFunctions[key] = []; - } - // eslint-disable-next-line prefer-spread - hookFunctions[key].push.apply(hookFunctions[key], pushHooks[key]); - } - } - - return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters); -} - -/** - * Returns WorkflowHooks instance for main process if workflow runs via worker - */ -export function getWorkflowHooksWorkerMain( - mode: WorkflowExecuteMode, - executionId: string, - workflowData: IWorkflowBase, - optionalParameters?: IWorkflowHooksOptionalParameters, -): WorkflowHooks { - optionalParameters = optionalParameters || {}; - const hookFunctions = hookFunctionsPreExecute(); - - // TODO: why are workers pushing to frontend? - // TODO: simplifying this for now to just leave the bare minimum hooks - - // const hookFunctions = hookFunctionsPush(); - // const preExecuteFunctions = hookFunctionsPreExecute(); - // for (const key of Object.keys(preExecuteFunctions)) { - // if (hookFunctions[key] === undefined) { - // hookFunctions[key] = []; - // } - // hookFunctions[key]!.push.apply(hookFunctions[key], preExecuteFunctions[key]); - // } - - // When running with worker mode, main process executes - // Only workflowExecuteBefore + workflowExecuteAfter - // So to avoid confusion, we are removing other hooks. - hookFunctions.nodeExecuteBefore = []; - hookFunctions.nodeExecuteAfter = []; - hookFunctions.workflowExecuteAfter = [ - async function (this: WorkflowHooks, fullRunData: IRun): Promise { - // Don't delete executions before they are finished - if (!fullRunData.finished) return; - - const executionStatus = determineFinalExecutionStatus(fullRunData); - fullRunData.status = executionStatus; - - const saveSettings = toSaveSettings(this.workflowData.settings); - - const shouldNotSave = - (executionStatus === 'success' && !saveSettings.success) || - (executionStatus !== 'success' && !saveSettings.error); - - if (shouldNotSave) { - await Container.get(ExecutionRepository).hardDelete({ - workflowId: this.workflowData.id, - executionId: this.executionId, - }); - } - }, - ]; - - return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters); -} - -/** - * Returns WorkflowHooks instance for running the main workflow - * - */ -export function getWorkflowHooksMain( - data: IWorkflowExecutionDataProcess, - executionId: string, -): WorkflowHooks { - const hookFunctions = hookFunctionsSave(); - const pushFunctions = hookFunctionsPush(); - for (const key of Object.keys(pushFunctions)) { - const hooks = hookFunctions[key] ?? []; - hooks.push.apply(hookFunctions[key], pushFunctions[key]); - } - - const preExecuteFunctions = hookFunctionsPreExecute(); - for (const key of Object.keys(preExecuteFunctions)) { - const hooks = hookFunctions[key] ?? []; - hooks.push.apply(hookFunctions[key], preExecuteFunctions[key]); - } - - if (!hookFunctions.nodeExecuteBefore) hookFunctions.nodeExecuteBefore = []; - if (!hookFunctions.nodeExecuteAfter) hookFunctions.nodeExecuteAfter = []; - - return new WorkflowHooks(hookFunctions, data.executionMode, executionId, data.workflowData, { - pushRef: data.pushRef, - retryOf: data.retryOf as string, - }); -} diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index a5ffb728d6..82beb20648 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -3,6 +3,7 @@ /* eslint-disable @typescript-eslint/no-shadow */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { Container, Service } from '@n8n/di'; +import type { ExecutionLifecycleHooks } from 'n8n-core'; import { ErrorReporter, InstanceSettings, Logger, WorkflowExecute } from 'n8n-core'; import type { ExecutionError, @@ -11,7 +12,6 @@ import type { IPinData, IRun, WorkflowExecuteMode, - WorkflowHooks, IWorkflowExecutionDataProcess, } from 'n8n-workflow'; import { ExecutionCancelledError, Workflow } from 'n8n-workflow'; @@ -20,7 +20,13 @@ import PCancelable from 'p-cancelable'; import { ActiveExecutions } from '@/active-executions'; import config from '@/config'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { ExternalHooks } from '@/external-hooks'; +import { ExecutionNotFoundError } from '@/errors/execution-not-found-error'; +import { + getLifecycleHooksForRegularMain, + getLifecycleHooksForScalingWorker, + getLifecycleHooksForScalingMain, +} from '@/execution-lifecycle/execution-lifecycle-hooks'; +import { ManualExecutionService } from '@/manual-execution.service'; import { NodeTypes } from '@/node-types'; import type { ScalingService } from '@/scaling/scaling.service'; import type { Job, JobData } from '@/scaling/scaling.types'; @@ -29,9 +35,7 @@ import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-da import { generateFailedExecutionFromError } from '@/workflow-helpers'; import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.service'; -import { ExecutionNotFoundError } from './errors/execution-not-found-error'; -import { EventService } from './events/event.service'; -import { ManualExecutionService } from './manual-execution.service'; +import { MaxStalledCountError } from './errors/max-stalled-count.error'; @Service() export class WorkflowRunner { @@ -44,11 +48,9 @@ export class WorkflowRunner { private readonly errorReporter: ErrorReporter, private readonly activeExecutions: ActiveExecutions, private readonly executionRepository: ExecutionRepository, - private readonly externalHooks: ExternalHooks, private readonly workflowStaticDataService: WorkflowStaticDataService, private readonly nodeTypes: NodeTypes, private readonly permissionChecker: PermissionChecker, - private readonly eventService: EventService, private readonly instanceSettings: InstanceSettings, private readonly manualExecutionService: ManualExecutionService, ) {} @@ -59,7 +61,7 @@ export class WorkflowRunner { startedAt: Date, executionMode: WorkflowExecuteMode, executionId: string, - hooks?: WorkflowHooks, + hooks?: ExecutionLifecycleHooks, ) { // This means the execution was probably cancelled and has already // been cleaned up. @@ -114,9 +116,7 @@ export class WorkflowRunner { // set the execution to failed. this.activeExecutions.finalizeExecution(executionId, fullRunData); - if (hooks) { - await hooks.executeHookFunctions('workflowExecuteAfter', [fullRunData]); - } + await hooks?.runHook('workflowExecuteAfter', [fullRunData]); } /** Run the workflow @@ -138,12 +138,9 @@ export class WorkflowRunner { } catch (error) { // Create a failed execution with the data for the node, save it and abort execution const runData = generateFailedExecutionFromError(data.executionMode, error, error.node); - const workflowHooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId); - await workflowHooks.executeHookFunctions('workflowExecuteBefore', [ - undefined, - data.executionData, - ]); - await workflowHooks.executeHookFunctions('workflowExecuteAfter', [runData]); + const lifecycleHooks = getLifecycleHooksForRegularMain(data, executionId); + await lifecycleHooks.runHook('workflowExecuteBefore', [undefined, data.executionData]); + await lifecycleHooks.runHook('workflowExecuteAfter', [runData]); responsePromise?.reject(error); this.activeExecutions.finalizeExecution(executionId); return executionId; @@ -163,7 +160,6 @@ export class WorkflowRunner { await this.enqueueExecution(executionId, data, loadStaticData, realtime); } else { await this.runMainProcess(executionId, data, loadStaticData, restartExecutionId); - this.eventService.emit('workflow-pre-execute', { executionId, data }); } // only run these when not in queue mode or when the execution is manual, @@ -174,35 +170,17 @@ export class WorkflowRunner { data.executionMode === 'manual' ) { const postExecutePromise = this.activeExecutions.getPostExecutePromise(executionId); - postExecutePromise - .then(async (executionData) => { - this.eventService.emit('workflow-post-execute', { - workflow: data.workflowData, - executionId, - userId: data.userId, - runData: executionData, - }); - if (this.externalHooks.exists('workflow.postExecute')) { - try { - await this.externalHooks.run('workflow.postExecute', [ - executionData, - data.workflowData, - executionId, - ]); - } catch (error) { - this.errorReporter.error(error); - this.logger.error('There was a problem running hook "workflow.postExecute"', error); - } - } - }) - .catch((error) => { - if (error instanceof ExecutionCancelledError) return; - this.errorReporter.error(error); - this.logger.error( - 'There was a problem running internal hook "onWorkflowPostExecute"', - error, - ); + postExecutePromise.catch((error) => { + if (error instanceof ExecutionCancelledError) return; + this.errorReporter.error(error, { + extra: { executionId, workflowId }, }); + this.logger.error('There was an error in the post-execution promise', { + error, + executionId, + workflowId, + }); + }); } return executionId; @@ -234,7 +212,7 @@ export class WorkflowRunner { } let pinData: IPinData | undefined; - if (data.executionMode === 'manual') { + if (['manual', 'evaluation'].includes(data.executionMode)) { pinData = data.pinData ?? data.workflowData.pinData; } @@ -267,13 +245,12 @@ export class WorkflowRunner { await this.executionRepository.setRunning(executionId); // write try { - additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId); + const lifecycleHooks = getLifecycleHooksForRegularMain(data, executionId); + additionalData.hooks = lifecycleHooks; - additionalData.hooks.hookFunctions.sendResponse = [ - async (response: IExecuteResponsePromiseData): Promise => { - this.activeExecutions.resolveResponsePromise(executionId, response); - }, - ]; + lifecycleHooks.addHandler('sendResponse', (response) => { + this.activeExecutions.resolveResponsePromise(executionId, response); + }); additionalData.setExecutionStatus = WorkflowExecuteAdditionalData.setExecutionStatus.bind({ executionId, @@ -364,30 +341,32 @@ export class WorkflowRunner { // TODO: For realtime jobs should probably also not do retry or not retry if they are older than x seconds. // Check if they get retried by default and how often. let job: Job; - let hooks: WorkflowHooks; + let lifecycleHooks: ExecutionLifecycleHooks; try { job = await this.scalingService.addJob(jobData, { priority: realtime ? 50 : 100 }); - hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerMain( + lifecycleHooks = getLifecycleHooksForScalingMain( data.executionMode, executionId, data.workflowData, - { retryOf: data.retryOf ? data.retryOf.toString() : undefined }, + { + retryOf: data.retryOf ?? undefined, + }, ); // Normally also workflow should be supplied here but as it only used for sending // data to editor-UI is not needed. - await hooks.executeHookFunctions('workflowExecuteBefore', [undefined, data.executionData]); + await lifecycleHooks.runHook('workflowExecuteBefore', [undefined, data.executionData]); } catch (error) { - // We use "getWorkflowHooksWorkerExecuter" as "getWorkflowHooksWorkerMain" does not contain the + // We use "getLifecycleHooksForScalingWorker" as "getLifecycleHooksForScalingMain" does not contain the // "workflowExecuteAfter" which we require. - const hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( + const lifecycleHooks = getLifecycleHooksForScalingWorker( data.executionMode, executionId, data.workflowData, - { retryOf: data.retryOf ? data.retryOf.toString() : undefined }, + { retryOf: data.retryOf ?? undefined }, ); - await this.processError(error, new Date(), data.executionMode, executionId, hooks); + await this.processError(error, new Date(), data.executionMode, executionId, lifecycleHooks); throw error; } @@ -397,17 +376,23 @@ export class WorkflowRunner { onCancel(async () => { await this.scalingService.stopJob(job); - // We use "getWorkflowHooksWorkerExecuter" as "getWorkflowHooksWorkerMain" does not contain the + // We use "getLifecycleHooksForScalingWorker" as "getLifecycleHooksForScalingMain" does not contain the // "workflowExecuteAfter" which we require. - const hooksWorker = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( + const lifecycleHooks = getLifecycleHooksForScalingWorker( data.executionMode, executionId, data.workflowData, - { retryOf: data.retryOf ? data.retryOf.toString() : undefined }, + { retryOf: data.retryOf ?? undefined }, ); const error = new ExecutionCancelledError(executionId); - await this.processError(error, new Date(), data.executionMode, executionId, hooksWorker); + await this.processError( + error, + new Date(), + data.executionMode, + executionId, + lifecycleHooks, + ); reject(error); }); @@ -415,15 +400,29 @@ export class WorkflowRunner { try { await job.finished(); } catch (error) { - // We use "getWorkflowHooksWorkerExecuter" as "getWorkflowHooksWorkerMain" does not contain the + if ( + error instanceof Error && + error.message.includes('job stalled more than maxStalledCount') + ) { + error = new MaxStalledCountError(error); + } + + // We use "getLifecycleHooksForScalingWorker" as "getLifecycleHooksForScalingMain" does not contain the // "workflowExecuteAfter" which we require. - const hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter( + const lifecycleHooks = getLifecycleHooksForScalingWorker( data.executionMode, executionId, data.workflowData, - { retryOf: data.retryOf ? data.retryOf.toString() : undefined }, + { retryOf: data.retryOf ?? undefined }, + ); + + await this.processError( + error, + new Date(), + data.executionMode, + executionId, + lifecycleHooks, ); - await this.processError(error, new Date(), data.executionMode, executionId, hooks); reject(error); } @@ -449,7 +448,7 @@ export class WorkflowRunner { // Normally also static data should be supplied here but as it only used for sending // data to editor-UI is not needed. - await hooks.executeHookFunctions('workflowExecuteAfter', [runData]); + await lifecycleHooks.runHook('workflowExecuteAfter', [runData]); resolve(runData); }, diff --git a/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts b/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts index 3d0bec39de..00a81700e0 100644 --- a/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts +++ b/packages/cli/src/workflows/__tests__/workflow-execution.service.test.ts @@ -1,11 +1,15 @@ import { mock } from 'jest-mock-extended'; -import type { INode } from 'n8n-workflow'; +import type { INode, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; +import type { User } from '@/databases/entities/user'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import type { IWorkflowDb } from '@/interfaces'; +import * as WorkflowExecuteAdditionalData from '@/workflow-execute-additional-data'; import type { WorkflowRunner } from '@/workflow-runner'; import { WorkflowExecutionService } from '@/workflows/workflow-execution.service'; +import type { WorkflowRequest } from '../workflow.request'; + const webhookNode: INode = { name: 'Webhook', type: 'n8n-nodes-base.webhook', @@ -63,6 +67,9 @@ describe('WorkflowExecutionService', () => { mock(), ); + const additionalData = mock({}); + jest.spyOn(WorkflowExecuteAdditionalData, 'getBase').mockResolvedValue(additionalData); + describe('runWorkflow()', () => { test('should call `WorkflowRunner.run()`', async () => { const node = mock(); @@ -76,6 +83,222 @@ describe('WorkflowExecutionService', () => { }); }); + describe('executeManually()', () => { + test('should call `WorkflowRunner.run()` with correct parameters with default partial execution logic', async () => { + const executionId = 'fake-execution-id'; + const userId = 'user-id'; + const user = mock({ id: userId }); + const runPayload = mock({ startNodes: [] }); + + workflowRunner.run.mockResolvedValue(executionId); + + const result = await workflowExecutionService.executeManually(runPayload, user); + + expect(workflowRunner.run).toHaveBeenCalledWith({ + destinationNode: runPayload.destinationNode, + executionMode: 'manual', + runData: runPayload.runData, + pinData: undefined, + pushRef: undefined, + workflowData: runPayload.workflowData, + userId, + partialExecutionVersion: 1, + startNodes: runPayload.startNodes, + dirtyNodeNames: runPayload.dirtyNodeNames, + triggerToStartFrom: runPayload.triggerToStartFrom, + }); + expect(result).toEqual({ executionId }); + }); + + [ + { + name: 'trigger', + type: 'n8n-nodes-base.airtableTrigger', + // Avoid mock constructor evaluated as true + disabled: undefined, + }, + { + name: 'webhook', + type: 'n8n-nodes-base.webhook', + disabled: undefined, + }, + ].forEach((triggerNode: Partial) => { + test(`should call WorkflowRunner.run() with pinned trigger with type ${triggerNode.name}`, async () => { + const additionalData = mock({}); + jest.spyOn(WorkflowExecuteAdditionalData, 'getBase').mockResolvedValue(additionalData); + const executionId = 'fake-execution-id'; + const userId = 'user-id'; + const user = mock({ id: userId }); + const runPayload = mock({ + startNodes: [], + workflowData: { + pinData: { + trigger: [{}], + }, + nodes: [triggerNode], + }, + triggerToStartFrom: undefined, + }); + + workflowRunner.run.mockResolvedValue(executionId); + + const result = await workflowExecutionService.executeManually(runPayload, user); + + expect(workflowRunner.run).toHaveBeenCalledWith({ + destinationNode: runPayload.destinationNode, + executionMode: 'manual', + runData: runPayload.runData, + pinData: runPayload.workflowData.pinData, + pushRef: undefined, + workflowData: runPayload.workflowData, + userId, + partialExecutionVersion: 1, + startNodes: [ + { + name: triggerNode.name, + sourceData: null, + }, + ], + dirtyNodeNames: runPayload.dirtyNodeNames, + triggerToStartFrom: runPayload.triggerToStartFrom, + }); + expect(result).toEqual({ executionId }); + }); + }); + + test('should start from pinned trigger', async () => { + const executionId = 'fake-execution-id'; + const userId = 'user-id'; + const user = mock({ id: userId }); + + const pinnedTrigger: INode = { + id: '1', + typeVersion: 1, + position: [1, 2], + parameters: {}, + name: 'pinned', + type: 'n8n-nodes-base.airtableTrigger', + }; + + const unexecutedTrigger: INode = { + id: '1', + typeVersion: 1, + position: [1, 2], + parameters: {}, + name: 'to-start-from', + type: 'n8n-nodes-base.airtableTrigger', + }; + + const runPayload: WorkflowRequest.ManualRunPayload = { + startNodes: [], + workflowData: { + id: 'abc', + name: 'test', + active: false, + pinData: { + [pinnedTrigger.name]: [{ json: {} }], + }, + nodes: [unexecutedTrigger, pinnedTrigger], + connections: {}, + createdAt: new Date(), + updatedAt: new Date(), + }, + runData: {}, + }; + + workflowRunner.run.mockResolvedValue(executionId); + + const result = await workflowExecutionService.executeManually(runPayload, user); + + expect(workflowRunner.run).toHaveBeenCalledWith({ + destinationNode: runPayload.destinationNode, + executionMode: 'manual', + runData: runPayload.runData, + pinData: runPayload.workflowData.pinData, + pushRef: undefined, + workflowData: runPayload.workflowData, + userId, + partialExecutionVersion: 1, + startNodes: [ + { + // Start from pinned trigger + name: pinnedTrigger.name, + sourceData: null, + }, + ], + dirtyNodeNames: runPayload.dirtyNodeNames, + // no trigger to start from + triggerToStartFrom: undefined, + }); + expect(result).toEqual({ executionId }); + }); + + test('should ignore pinned trigger and start from unexecuted trigger', async () => { + const executionId = 'fake-execution-id'; + const userId = 'user-id'; + const user = mock({ id: userId }); + + const pinnedTrigger: INode = { + id: '1', + typeVersion: 1, + position: [1, 2], + parameters: {}, + name: 'pinned', + type: 'n8n-nodes-base.airtableTrigger', + }; + + const unexecutedTrigger: INode = { + id: '1', + typeVersion: 1, + position: [1, 2], + parameters: {}, + name: 'to-start-from', + type: 'n8n-nodes-base.airtableTrigger', + }; + + const runPayload: WorkflowRequest.ManualRunPayload = { + startNodes: [], + workflowData: { + id: 'abc', + name: 'test', + active: false, + pinData: { + [pinnedTrigger.name]: [{ json: {} }], + }, + nodes: [unexecutedTrigger, pinnedTrigger], + connections: {}, + createdAt: new Date(), + updatedAt: new Date(), + }, + runData: {}, + triggerToStartFrom: { + name: unexecutedTrigger.name, + }, + }; + + workflowRunner.run.mockResolvedValue(executionId); + + const result = await workflowExecutionService.executeManually(runPayload, user); + + expect(workflowRunner.run).toHaveBeenCalledWith({ + destinationNode: runPayload.destinationNode, + executionMode: 'manual', + runData: runPayload.runData, + pinData: runPayload.workflowData.pinData, + pushRef: undefined, + workflowData: runPayload.workflowData, + userId, + partialExecutionVersion: 1, + // ignore pinned trigger + startNodes: [], + dirtyNodeNames: runPayload.dirtyNodeNames, + // pass unexecuted trigger to start from + triggerToStartFrom: runPayload.triggerToStartFrom, + }); + expect(result).toEqual({ executionId }); + }); + }); + describe('selectPinnedActivatorStarter()', () => { const workflow = mock({ nodes: [], diff --git a/packages/cli/src/workflows/workflow-execution.service.ts b/packages/cli/src/workflows/workflow-execution.service.ts index bccd5c1f13..23394493df 100644 --- a/packages/cli/src/workflows/workflow-execution.service.ts +++ b/packages/cli/src/workflows/workflow-execution.service.ts @@ -97,15 +97,21 @@ export class WorkflowExecutionService { }: WorkflowRequest.ManualRunPayload, user: User, pushRef?: string, - partialExecutionVersion?: string, + partialExecutionVersion: 1 | 2 = 1, ) { const pinData = workflowData.pinData; - const pinnedTrigger = this.selectPinnedActivatorStarter( + let pinnedTrigger = this.selectPinnedActivatorStarter( workflowData, startNodes?.map((nodeData) => nodeData.name), pinData, ); + // if we have a trigger to start from and it's not the pinned trigger + // ignore the pinned trigger + if (pinnedTrigger && triggerToStartFrom && pinnedTrigger.name !== triggerToStartFrom.name) { + pinnedTrigger = null; + } + // If webhooks nodes exist and are active we have to wait for till we receive a call if ( pinnedTrigger === null && @@ -142,7 +148,7 @@ export class WorkflowExecutionService { startNodes, workflowData, userId: user.id, - partialExecutionVersion: partialExecutionVersion ?? '0', + partialExecutionVersion, dirtyNodeNames, triggerToStartFrom, }; diff --git a/packages/cli/src/workflows/workflow-sharing.service.ts b/packages/cli/src/workflows/workflow-sharing.service.ts index 220c9adaf9..d10ad45083 100644 --- a/packages/cli/src/workflows/workflow-sharing.service.ts +++ b/packages/cli/src/workflows/workflow-sharing.service.ts @@ -1,9 +1,9 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Service } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In } from '@n8n/typeorm'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { WorkflowSharingRole } from '@/databases/entities/shared-workflow'; import type { User } from '@/databases/entities/user'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; diff --git a/packages/cli/src/workflows/workflow.request.ts b/packages/cli/src/workflows/workflow.request.ts index 47fd2cdb93..3e44d5b950 100644 --- a/packages/cli/src/workflows/workflow.request.ts +++ b/packages/cli/src/workflows/workflow.request.ts @@ -55,12 +55,7 @@ export declare namespace WorkflowRequest { type NewName = AuthenticatedRequest<{}, {}, {}, { name?: string }>; - type ManualRun = AuthenticatedRequest< - { workflowId: string }, - {}, - ManualRunPayload, - { partialExecutionVersion?: string } - >; + type ManualRun = AuthenticatedRequest<{ workflowId: string }, {}, ManualRunPayload, {}>; type Share = AuthenticatedRequest<{ workflowId: string }, {}, { shareWithIds: string[] }>; diff --git a/packages/cli/src/workflows/workflow.service.ts b/packages/cli/src/workflows/workflow.service.ts index 2141e79ed5..fdb53c1832 100644 --- a/packages/cli/src/workflows/workflow.service.ts +++ b/packages/cli/src/workflows/workflow.service.ts @@ -1,3 +1,4 @@ +import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import @@ -54,6 +55,7 @@ export class WorkflowService { private readonly projectService: ProjectService, private readonly executionRepository: ExecutionRepository, private readonly eventService: EventService, + private readonly globalConfig: GlobalConfig, ) {} async getMany(user: User, options?: ListQuery.Options, includeScopes?: boolean) { @@ -202,7 +204,9 @@ export class WorkflowService { ]), ); - if (tagIds && !config.getEnv('workflowTagsDisabled')) { + const tagsDisabled = this.globalConfig.tags.disabled; + + if (tagIds && !tagsDisabled) { await this.workflowTagMappingRepository.overwriteTaggings(workflowId, tagIds); } @@ -210,7 +214,7 @@ export class WorkflowService { await this.workflowHistoryService.saveVersion(user, workflowUpdateData, workflowId); } - const relations = config.getEnv('workflowTagsDisabled') ? [] : ['tags']; + const relations = tagsDisabled ? [] : ['tags']; // We sadly get nothing back from "update". Neither if it updated a record // nor the new value. So query now the hopefully updated entry. @@ -268,6 +272,13 @@ export class WorkflowService { return updatedWorkflow; } + /** + * Deletes a workflow and returns it. + * + * If the workflow is active this will deactivate the workflow. + * If the user does not have the permissions to delete the workflow this does + * nothing and returns void. + */ async delete(user: User, workflowId: string): Promise { await this.externalHooks.run('workflow.delete', [workflowId]); diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index 865b38450e..76904e5201 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -1,4 +1,4 @@ -import { ImportWorkflowFromUrlDto } from '@n8n/api-types'; +import { ImportWorkflowFromUrlDto, ManualRunQueryDto } from '@n8n/api-types'; import { GlobalConfig } from '@n8n/config'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { In, type FindOptionsRelations } from '@n8n/typeorm'; @@ -9,7 +9,6 @@ import { ApplicationError } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; import { z } from 'zod'; -import config from '@/config'; import type { Project } from '@/databases/entities/project'; import { SharedWorkflow } from '@/databases/entities/shared-workflow'; import { WorkflowEntity } from '@/databases/entities/workflow-entity'; @@ -89,7 +88,7 @@ export class WorkflowsController { const { tags: tagIds } = req.body; - if (tagIds?.length && !config.getEnv('workflowTagsDisabled')) { + if (tagIds?.length && !this.globalConfig.tags.disabled) { newWorkflow.tags = await this.tagRepository.findMany(tagIds); } @@ -164,7 +163,7 @@ export class WorkflowsController { await this.workflowHistoryService.saveVersion(req.user, savedWorkflow, savedWorkflow.id); - if (tagIds && !config.getEnv('workflowTagsDisabled') && savedWorkflow.tags) { + if (tagIds && !this.globalConfig.tags.disabled && savedWorkflow.tags) { savedWorkflow.tags = this.tagService.sortByRequestOrder(savedWorkflow.tags, { requestOrder: tagIds, }); @@ -260,7 +259,7 @@ export class WorkflowsController { }, }; - if (!config.getEnv('workflowTagsDisabled')) { + if (!this.globalConfig.tags.disabled) { relations.tags = true; } @@ -268,7 +267,7 @@ export class WorkflowsController { workflowId, req.user, ['workflow:read'], - { includeTags: !config.getEnv('workflowTagsDisabled') }, + { includeTags: !this.globalConfig.tags.disabled }, ); if (!workflow) { @@ -296,7 +295,7 @@ export class WorkflowsController { workflowId, req.user, ['workflow:read'], - { includeTags: !config.getEnv('workflowTagsDisabled') }, + { includeTags: !this.globalConfig.tags.disabled }, ); if (!workflow) { @@ -367,7 +366,11 @@ export class WorkflowsController { @Post('/:workflowId/run') @ProjectScope('workflow:execute') - async runManually(req: WorkflowRequest.ManualRun) { + async runManually( + req: WorkflowRequest.ManualRun, + _res: unknown, + @Query query: ManualRunQueryDto, + ) { if (!req.body.workflowData.id) { throw new ApplicationError('You cannot execute a workflow without an ID', { level: 'warning', @@ -395,9 +398,7 @@ export class WorkflowsController { req.body, req.user, req.headers['push-ref'], - req.query.partialExecutionVersion === '-1' - ? config.getEnv('featureFlags.partialExecutionVersionDefault') - : req.query.partialExecutionVersion, + query.partialExecutionVersion, ); } diff --git a/packages/cli/templates/form-trigger-completion.handlebars b/packages/cli/templates/form-trigger-completion.handlebars index a15855d371..880a7f91d1 100644 --- a/packages/cli/templates/form-trigger-completion.handlebars +++ b/packages/cli/templates/form-trigger-completion.handlebars @@ -26,49 +26,53 @@ -

-
-
-
-

{{title}}

-

{{message}}

+ {{#if responseText}} + {{{responseText}}} + {{else}} +
+
+
+
+

{{title}}

+

{{message}}

+
-
- {{#if appendAttribution}} - - {{/if}} -
-
+ {{#if appendAttribution}} + + {{/if}} + +
+ {{/if}} diff --git a/packages/cli/templates/form-trigger.handlebars b/packages/cli/templates/form-trigger.handlebars index ee868b072f..18324e6dc7 100644 --- a/packages/cli/templates/form-trigger.handlebars +++ b/packages/cli/templates/form-trigger.handlebars @@ -2,12 +2,13 @@ + + + + + - + {{formTitle}} + + + +
+
+
+ + + + +
+
+

Connection successful

+
+
+

This window will close automatically in 5 seconds.

+
+
+
+ + diff --git a/packages/cli/test/integration/active-workflow-manager.test.ts b/packages/cli/test/integration/active-workflow-manager.test.ts index 3c98c2a4f1..8c502bba94 100644 --- a/packages/cli/test/integration/active-workflow-manager.test.ts +++ b/packages/cli/test/integration/active-workflow-manager.test.ts @@ -76,10 +76,7 @@ describe('init()', () => { it('should call external hook', async () => { await activeWorkflowManager.init(); - const [hook, arg] = externalHooks.run.mock.calls[0]; - - expect(hook).toBe('activeWorkflows.initialized'); - expect(arg).toBeEmptyArray(); + expect(externalHooks.run).toHaveBeenCalledWith('activeWorkflows.initialized'); }); it('should check that workflow can be activated', async () => { diff --git a/packages/cli/test/integration/api-keys.api.test.ts b/packages/cli/test/integration/api-keys.api.test.ts index 14050b543a..0f0dbf4e9e 100644 --- a/packages/cli/test/integration/api-keys.api.test.ts +++ b/packages/cli/test/integration/api-keys.api.test.ts @@ -1,9 +1,10 @@ +import type { ApiKeyWithRawValue } from '@n8n/api-types'; import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; -import type { ApiKey } from '@/databases/entities/api-key'; import type { User } from '@/databases/entities/user'; import { ApiKeyRepository } from '@/databases/repositories/api-key.repository'; +import { License } from '@/license'; import { PublicApiKeyService } from '@/services/public-api-key.service'; import { mockInstance } from '@test/mocking'; @@ -13,6 +14,10 @@ import * as testDb from './shared/test-db'; import type { SuperAgentTest } from './shared/types'; import * as utils from './shared/utils/'; +const license = mockInstance(License); + +license.getApiKeysPerUserLimit.mockImplementation(() => 2); + const testServer = utils.setupTestServer({ endpointGroups: ['apiKeys'] }); let publicApiKeyService: PublicApiKeyService; @@ -56,10 +61,13 @@ describe('Owner shell', () => { ownerShell = await createUserShell('global:owner'); }); - test('POST /api-keys should create an api key', async () => { - const newApiKeyResponse = await testServer.authAgentFor(ownerShell).post('/api-keys'); + test('POST /api-keys should create an api key with no expiration', async () => { + const newApiKeyResponse = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); - const newApiKey = newApiKeyResponse.body.data as ApiKey; + const newApiKey = newApiKeyResponse.body.data as ApiKeyWithRawValue; expect(newApiKeyResponse.statusCode).toBe(200); expect(newApiKey).toBeDefined(); @@ -72,31 +80,99 @@ describe('Owner shell', () => { id: expect.any(String), label: 'My API Key', userId: ownerShell.id, - apiKey: newApiKey.apiKey, + apiKey: newApiKey.rawApiKey, createdAt: expect.any(Date), updatedAt: expect.any(Date), }); + + expect(newApiKey.expiresAt).toBeNull(); + expect(newApiKey.rawApiKey).toBeDefined(); + }); + + test('POST /api-keys should create an api key with expiration', async () => { + const expiresAt = Date.now() + 1000; + + const newApiKeyResponse = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt }); + + const newApiKey = newApiKeyResponse.body.data as ApiKeyWithRawValue; + + expect(newApiKeyResponse.statusCode).toBe(200); + expect(newApiKey).toBeDefined(); + + const newStoredApiKey = await Container.get(ApiKeyRepository).findOneByOrFail({ + userId: ownerShell.id, + }); + + expect(newStoredApiKey).toEqual({ + id: expect.any(String), + label: 'My API Key', + userId: ownerShell.id, + apiKey: newApiKey.rawApiKey, + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + }); + + expect(newApiKey.expiresAt).toBe(expiresAt); + expect(newApiKey.rawApiKey).toBeDefined(); + }); + + test('POST /api-keys should fail if max number of API keys reached', async () => { + await testServer.authAgentFor(ownerShell).post('/api-keys').send({ label: 'My API Key' }); + + const secondApiKey = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key' }); + + expect(secondApiKey.statusCode).toBe(400); }); test('GET /api-keys should fetch the api key redacted', async () => { - const newApiKeyResponse = await testServer.authAgentFor(ownerShell).post('/api-keys'); + const expirationDateInTheFuture = Date.now() + 1000; + + const apiKeyWithNoExpiration = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); + + const apiKeyWithExpiration = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key 2', expiresAt: expirationDateInTheFuture }); const retrieveAllApiKeysResponse = await testServer.authAgentFor(ownerShell).get('/api-keys'); expect(retrieveAllApiKeysResponse.statusCode).toBe(200); - expect(retrieveAllApiKeysResponse.body.data[0]).toEqual({ - id: newApiKeyResponse.body.data.id, - label: 'My API Key', + expect(retrieveAllApiKeysResponse.body.data[1]).toEqual({ + id: apiKeyWithExpiration.body.data.id, + label: 'My API Key 2', userId: ownerShell.id, - apiKey: publicApiKeyService.redactApiKey(newApiKeyResponse.body.data.apiKey), + apiKey: publicApiKeyService.redactApiKey(apiKeyWithExpiration.body.data.rawApiKey), createdAt: expect.any(String), updatedAt: expect.any(String), + expiresAt: expirationDateInTheFuture, + }); + + expect(retrieveAllApiKeysResponse.body.data[0]).toEqual({ + id: apiKeyWithNoExpiration.body.data.id, + label: 'My API Key', + userId: ownerShell.id, + apiKey: publicApiKeyService.redactApiKey(apiKeyWithNoExpiration.body.data.rawApiKey), + createdAt: expect.any(String), + updatedAt: expect.any(String), + expiresAt: null, }); }); test('DELETE /api-keys/:id should delete the api key', async () => { - const newApiKeyResponse = await testServer.authAgentFor(ownerShell).post('/api-keys'); + const newApiKeyResponse = await testServer + .authAgentFor(ownerShell) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); const deleteApiKeyResponse = await testServer .authAgentFor(ownerShell) @@ -121,8 +197,11 @@ describe('Member', () => { await utils.setInstanceOwnerSetUp(true); }); - test('POST /api-keys should create an api key', async () => { - const newApiKeyResponse = await testServer.authAgentFor(member).post('/api-keys'); + test('POST /api-keys should create an api key with no expiration', async () => { + const newApiKeyResponse = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); expect(newApiKeyResponse.statusCode).toBe(200); expect(newApiKeyResponse.body.data.apiKey).toBeDefined(); @@ -136,35 +215,99 @@ describe('Member', () => { id: expect.any(String), label: 'My API Key', userId: member.id, - apiKey: newApiKeyResponse.body.data.apiKey, + apiKey: newApiKeyResponse.body.data.rawApiKey, createdAt: expect.any(Date), updatedAt: expect.any(Date), }); + + expect(newApiKeyResponse.body.data.expiresAt).toBeNull(); + expect(newApiKeyResponse.body.data.rawApiKey).toBeDefined(); + }); + + test('POST /api-keys should create an api key with expiration', async () => { + const expiresAt = Date.now() + 1000; + + const newApiKeyResponse = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt }); + + const newApiKey = newApiKeyResponse.body.data as ApiKeyWithRawValue; + + expect(newApiKeyResponse.statusCode).toBe(200); + expect(newApiKey).toBeDefined(); + + const newStoredApiKey = await Container.get(ApiKeyRepository).findOneByOrFail({ + userId: member.id, + }); + + expect(newStoredApiKey).toEqual({ + id: expect.any(String), + label: 'My API Key', + userId: member.id, + apiKey: newApiKey.rawApiKey, + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + }); + + expect(newApiKey.expiresAt).toBe(expiresAt); + expect(newApiKey.rawApiKey).toBeDefined(); + }); + + test('POST /api-keys should fail if max number of API keys reached', async () => { + await testServer.authAgentFor(member).post('/api-keys').send({ label: 'My API Key' }); + + const secondApiKey = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key' }); + + expect(secondApiKey.statusCode).toBe(400); }); test('GET /api-keys should fetch the api key redacted', async () => { - const newApiKeyResponse = await testServer.authAgentFor(member).post('/api-keys'); + const expirationDateInTheFuture = Date.now() + 1000; + + const apiKeyWithNoExpiration = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); + + const apiKeyWithExpiration = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key 2', expiresAt: expirationDateInTheFuture }); const retrieveAllApiKeysResponse = await testServer.authAgentFor(member).get('/api-keys'); expect(retrieveAllApiKeysResponse.statusCode).toBe(200); - expect(retrieveAllApiKeysResponse.body.data[0]).toEqual({ - id: newApiKeyResponse.body.data.id, - label: 'My API Key', + expect(retrieveAllApiKeysResponse.body.data[1]).toEqual({ + id: apiKeyWithExpiration.body.data.id, + label: 'My API Key 2', userId: member.id, - apiKey: publicApiKeyService.redactApiKey(newApiKeyResponse.body.data.apiKey), + apiKey: publicApiKeyService.redactApiKey(apiKeyWithExpiration.body.data.rawApiKey), createdAt: expect.any(String), updatedAt: expect.any(String), + expiresAt: expirationDateInTheFuture, }); - expect(newApiKeyResponse.body.data.apiKey).not.toEqual( - retrieveAllApiKeysResponse.body.data[0].apiKey, - ); + expect(retrieveAllApiKeysResponse.body.data[0]).toEqual({ + id: apiKeyWithNoExpiration.body.data.id, + label: 'My API Key', + userId: member.id, + apiKey: publicApiKeyService.redactApiKey(apiKeyWithNoExpiration.body.data.rawApiKey), + createdAt: expect.any(String), + updatedAt: expect.any(String), + expiresAt: null, + }); }); test('DELETE /api-keys/:id should delete the api key', async () => { - const newApiKeyResponse = await testServer.authAgentFor(member).post('/api-keys'); + const newApiKeyResponse = await testServer + .authAgentFor(member) + .post('/api-keys') + .send({ label: 'My API Key', expiresAt: null }); const deleteApiKeyResponse = await testServer .authAgentFor(member) diff --git a/packages/cli/test/integration/commands/worker.cmd.test.ts b/packages/cli/test/integration/commands/worker.cmd.test.ts index 8b9cebe854..36d6322ec7 100644 --- a/packages/cli/test/integration/commands/worker.cmd.test.ts +++ b/packages/cli/test/integration/commands/worker.cmd.test.ts @@ -17,8 +17,8 @@ import { Publisher } from '@/scaling/pubsub/publisher.service'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import { ScalingService } from '@/scaling/scaling.service'; import { OrchestrationService } from '@/services/orchestration.service'; +import { TaskBrokerServer } from '@/task-runners/task-broker/task-broker-server'; import { TaskRunnerProcess } from '@/task-runners/task-runner-process'; -import { TaskRunnerServer } from '@/task-runners/task-runner-server'; import { Telemetry } from '@/telemetry'; import { setupTestCommand } from '@test-integration/utils/test-command'; @@ -36,7 +36,7 @@ const messageEventBus = mockInstance(MessageEventBus); const logStreamingEventRelay = mockInstance(LogStreamingEventRelay); const scalingService = mockInstance(ScalingService); const orchestrationService = mockInstance(OrchestrationService); -const taskRunnerServer = mockInstance(TaskRunnerServer); +const taskBrokerServer = mockInstance(TaskBrokerServer); const taskRunnerProcess = mockInstance(TaskRunnerProcess); mockInstance(Publisher); mockInstance(Subscriber); @@ -60,7 +60,7 @@ test('worker initializes all its components', async () => { expect(logStreamingEventRelay.init).toHaveBeenCalledTimes(1); expect(orchestrationService.init).toHaveBeenCalledTimes(1); expect(messageEventBus.send).toHaveBeenCalledTimes(1); - expect(taskRunnerServer.start).toHaveBeenCalledTimes(1); + expect(taskBrokerServer.start).toHaveBeenCalledTimes(1); expect(taskRunnerProcess.start).toHaveBeenCalledTimes(1); expect(config.getEnv('executions.mode')).toBe('queue'); diff --git a/packages/cli/test/integration/credentials/credentials.api.ee.test.ts b/packages/cli/test/integration/credentials/credentials.api.ee.test.ts index 090de90603..98cecb6664 100644 --- a/packages/cli/test/integration/credentials/credentials.api.ee.test.ts +++ b/packages/cli/test/integration/credentials/credentials.api.ee.test.ts @@ -1,10 +1,10 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { In } from '@n8n/typeorm'; import config from '@/config'; import { CredentialsService } from '@/credentials/credentials.service'; import type { Project } from '@/databases/entities/project'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { User } from '@/databases/entities/user'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository'; @@ -226,12 +226,12 @@ describe('GET /credentials', () => { // // ARRANGE // - const project1 = await projectService.createTeamProject('Team Project', member); + const project1 = await projectService.createTeamProject(member, { name: 'Team Project' }); await projectService.addUser(project1.id, anotherMember.id, 'project:editor'); // anotherMember should see this one const credential1 = await saveCredential(randomCredentialPayload(), { project: project1 }); - const project2 = await projectService.createTeamProject('Team Project', member); + const project2 = await projectService.createTeamProject(member, { name: 'Team Project' }); // anotherMember should NOT see this one await saveCredential(randomCredentialPayload(), { project: project2 }); diff --git a/packages/cli/test/integration/credentials/credentials.api.test.ts b/packages/cli/test/integration/credentials/credentials.api.test.ts index 7bb0a8918a..ece8340e83 100644 --- a/packages/cli/test/integration/credentials/credentials.api.test.ts +++ b/packages/cli/test/integration/credentials/credentials.api.test.ts @@ -4,6 +4,7 @@ import type { Scope } from '@sentry/node'; import { Credentials } from 'n8n-core'; import { randomString } from 'n8n-workflow'; +import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; import { CredentialsService } from '@/credentials/credentials.service'; import type { Project } from '@/databases/entities/project'; import type { User } from '@/databases/entities/user'; @@ -632,25 +633,25 @@ describe('GET /credentials', () => { expect(response.body.data.map((credential) => credential.id)).toContain(memberCredential.id); }); - test('should return all credentials to instance owners when working on their own personal project', async () => { + test('should not ignore the project filter when the request is done by an owner and also includes the scopes', async () => { const ownerCredential = await saveCredential(payload(), { user: owner, role: 'credential:owner', }); - const memberCredential = await saveCredential(payload(), { - user: member, - role: 'credential:owner', - }); + // should not show up + await saveCredential(payload(), { user: member, role: 'credential:owner' }); const response: GetAllResponse = await testServer .authAgentFor(owner) .get('/credentials') - .query(`filter={ "projectId": "${ownerPersonalProject.id}" }&includeScopes=true`) + .query({ + filter: JSON.stringify({ projectId: ownerPersonalProject.id }), + includeScopes: true, + }) .expect(200); - expect(response.body.data).toHaveLength(2); - expect(response.body.data.map((credential) => credential.id)).toContain(ownerCredential.id); - expect(response.body.data.map((credential) => credential.id)).toContain(memberCredential.id); + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0].id).toBe(ownerCredential.id); }); }); @@ -1164,6 +1165,73 @@ describe('PATCH /credentials/:id', () => { expect(shellCredential.name).toBe(patchPayload.name); // updated }); + test('should not store redacted value in the db for oauthTokenData', async () => { + // ARRANGE + const credentialService = Container.get(CredentialsService); + const redactSpy = jest.spyOn(credentialService, 'redact').mockReturnValueOnce({ + accessToken: CREDENTIAL_BLANKING_VALUE, + oauthTokenData: CREDENTIAL_BLANKING_VALUE, + }); + + const payload = randomCredentialPayload(); + payload.data.oauthTokenData = { tokenData: true }; + const savedCredential = await saveCredential(payload, { + user: owner, + role: 'credential:owner', + }); + + // ACT + const patchPayload = { ...payload, data: { foo: 'bar' } }; + await authOwnerAgent.patch(`/credentials/${savedCredential.id}`).send(patchPayload).expect(200); + + // ASSERT + const response = await authOwnerAgent + .get(`/credentials/${savedCredential.id}`) + .query({ includeData: true }) + .expect(200); + + const { id, data } = response.body.data; + + expect(id).toBe(savedCredential.id); + expect(data).toEqual({ + ...patchPayload.data, + // should be the original + oauthTokenData: payload.data.oauthTokenData, + }); + expect(redactSpy).not.toHaveBeenCalled(); + }); + + test('should not allow to overwrite oauthTokenData', async () => { + // ARRANGE + const payload = randomCredentialPayload(); + payload.data.oauthTokenData = { tokenData: true }; + const savedCredential = await saveCredential(payload, { + user: owner, + role: 'credential:owner', + }); + + // ACT + const patchPayload = { + ...payload, + data: { accessToken: 'new', oauthTokenData: { tokenData: false } }, + }; + await authOwnerAgent.patch(`/credentials/${savedCredential.id}`).send(patchPayload).expect(200); + + // ASSERT + const response = await authOwnerAgent + .get(`/credentials/${savedCredential.id}`) + .query({ includeData: true }) + .expect(200); + + const { id, data } = response.body.data; + + expect(id).toBe(savedCredential.id); + // was overwritten + expect(data.accessToken).toBe(patchPayload.data.accessToken); + // was not overwritten + expect(data.oauthTokenData).toEqual(payload.data.oauthTokenData); + }); + test('should fail with invalid inputs', async () => { const savedCredential = await saveCredential(randomCredentialPayload(), { user: owner, diff --git a/packages/cli/test/integration/database/repositories/workflow.repository.test.ts b/packages/cli/test/integration/database/repositories/workflow.repository.test.ts index 63b87df7e8..c92dcdde0a 100644 --- a/packages/cli/test/integration/database/repositories/workflow.repository.test.ts +++ b/packages/cli/test/integration/database/repositories/workflow.repository.test.ts @@ -73,7 +73,7 @@ describe('WorkflowRepository', () => { }); describe('getActiveIds', () => { - it('should return active workflow IDs', async () => { + it('should return all active workflow IDs when invoked without maxResults', async () => { // // ARRANGE // @@ -92,6 +92,28 @@ describe('WorkflowRepository', () => { // ASSERT // expect(activeIds).toEqual([workflows[0].id]); + expect(activeIds).toHaveLength(1); + }); + + it('should return a capped number of active workflow IDs when invoked with maxResults', async () => { + // + // ARRANGE + // + await Promise.all([ + createWorkflow({ active: true }), + createWorkflow({ active: false }), + createWorkflow({ active: true }), + ]); + + // + // ACT + // + const activeIds = await Container.get(WorkflowRepository).getActiveIds({ maxResults: 1 }); + + // + // ASSERT + // + expect(activeIds).toHaveLength(1); }); }); }); diff --git a/packages/cli/test/integration/environments/source-control-import.service.test.ts b/packages/cli/test/integration/environments/source-control-import.service.test.ts index 6835d18f58..5bfd5d0e79 100644 --- a/packages/cli/test/integration/environments/source-control-import.service.test.ts +++ b/packages/cli/test/integration/environments/source-control-import.service.test.ts @@ -1,3 +1,4 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { mock } from 'jest-mock-extended'; import { Cipher } from 'n8n-core'; @@ -9,9 +10,9 @@ import fsp from 'node:fs/promises'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository'; +import { UserRepository } from '@/databases/repositories/user.repository'; import { SourceControlImportService } from '@/environments.ee/source-control/source-control-import.service.ee'; import type { ExportableCredential } from '@/environments.ee/source-control/types/exportable-credential'; -import type { SourceControlledFile } from '@/environments.ee/source-control/types/source-controlled-file'; import { mockInstance } from '../../shared/mocking'; import { saveCredential } from '../shared/db/credentials'; @@ -21,11 +22,32 @@ import { randomCredentialPayload } from '../shared/random'; import * as testDb from '../shared/test-db'; describe('SourceControlImportService', () => { + let credentialsRepository: CredentialsRepository; + let projectRepository: ProjectRepository; + let sharedCredentialsRepository: SharedCredentialsRepository; + let userRepository: UserRepository; let service: SourceControlImportService; const cipher = mockInstance(Cipher); beforeAll(async () => { + await testDb.init(); + + credentialsRepository = Container.get(CredentialsRepository); + projectRepository = Container.get(ProjectRepository); + sharedCredentialsRepository = Container.get(SharedCredentialsRepository); + userRepository = Container.get(UserRepository); service = new SourceControlImportService( + mock(), + mock(), + mock(), + mock(), + credentialsRepository, + projectRepository, + mock(), + mock(), + sharedCredentialsRepository, + userRepository, + mock(), mock(), mock(), mock(), @@ -33,8 +55,6 @@ describe('SourceControlImportService', () => { mock(), mock({ n8nFolder: '/some-path' }), ); - - await testDb.init(); }); afterEach(async () => { @@ -75,7 +95,7 @@ describe('SourceControlImportService', () => { const personalProject = await getPersonalProject(member); - const sharing = await Container.get(SharedCredentialsRepository).findOneBy({ + const sharing = await sharedCredentialsRepository.findOneBy({ credentialsId: CREDENTIAL_ID, projectId: personalProject.id, role: 'credential:owner', @@ -112,7 +132,7 @@ describe('SourceControlImportService', () => { const personalProject = await getPersonalProject(importingUser); - const sharing = await Container.get(SharedCredentialsRepository).findOneBy({ + const sharing = await sharedCredentialsRepository.findOneBy({ credentialsId: CREDENTIAL_ID, projectId: personalProject.id, role: 'credential:owner', @@ -149,7 +169,7 @@ describe('SourceControlImportService', () => { const personalProject = await getPersonalProject(importingUser); - const sharing = await Container.get(SharedCredentialsRepository).findOneBy({ + const sharing = await sharedCredentialsRepository.findOneBy({ credentialsId: CREDENTIAL_ID, projectId: personalProject.id, role: 'credential:owner', @@ -190,7 +210,7 @@ describe('SourceControlImportService', () => { const personalProject = await getPersonalProject(importingUser); - const sharing = await Container.get(SharedCredentialsRepository).findOneBy({ + const sharing = await sharedCredentialsRepository.findOneBy({ credentialsId: CREDENTIAL_ID, projectId: personalProject.id, role: 'credential:owner', @@ -223,7 +243,7 @@ describe('SourceControlImportService', () => { cipher.encrypt.mockReturnValue('some-encrypted-data'); { - const project = await Container.get(ProjectRepository).findOne({ + const project = await projectRepository.findOne({ where: [ { id: '1234-asdf', @@ -241,7 +261,7 @@ describe('SourceControlImportService', () => { importingUser.id, ); - const sharing = await Container.get(SharedCredentialsRepository).findOne({ + const sharing = await sharedCredentialsRepository.findOne({ where: { credentialsId: CREDENTIAL_ID, role: 'credential:owner', @@ -288,7 +308,7 @@ describe('SourceControlImportService', () => { importingUser.id, ); - const sharing = await Container.get(SharedCredentialsRepository).findOneBy({ + const sharing = await sharedCredentialsRepository.findOneBy({ credentialsId: CREDENTIAL_ID, projectId: project.id, role: 'credential:owner', @@ -332,7 +352,7 @@ describe('SourceControlImportService', () => { ); await expect( - Container.get(SharedCredentialsRepository).findBy({ + sharedCredentialsRepository.findBy({ credentialsId: credential.id, }), ).resolves.toMatchObject([ @@ -342,7 +362,7 @@ describe('SourceControlImportService', () => { }, ]); await expect( - Container.get(CredentialsRepository).findBy({ + credentialsRepository.findBy({ id: credential.id, }), ).resolves.toMatchObject([ diff --git a/packages/cli/test/integration/environments/source-control.test.ts b/packages/cli/test/integration/environments/source-control.api.test.ts similarity index 73% rename from packages/cli/test/integration/environments/source-control.test.ts rename to packages/cli/test/integration/environments/source-control.api.test.ts index 11a7ad8a2b..6ad771493b 100644 --- a/packages/cli/test/integration/environments/source-control.test.ts +++ b/packages/cli/test/integration/environments/source-control.api.test.ts @@ -1,10 +1,9 @@ +import type { SourceControlledFile } from '@n8n/api-types'; import { Container } from '@n8n/di'; -import config from '@/config'; import type { User } from '@/databases/entities/user'; import { SourceControlPreferencesService } from '@/environments.ee/source-control/source-control-preferences.service.ee'; import { SourceControlService } from '@/environments.ee/source-control/source-control.service.ee'; -import type { SourceControlledFile } from '@/environments.ee/source-control/types/source-controlled-file'; import { Telemetry } from '@/telemetry'; import { mockInstance } from '@test/mocking'; @@ -21,11 +20,17 @@ const testServer = utils.setupTestServer({ enabledFeatures: ['feat:sourceControl', 'feat:sharing'], }); +let sourceControlPreferencesService: SourceControlPreferencesService; + beforeAll(async () => { owner = await createUser({ role: 'global:owner' }); authOwnerAgent = testServer.authAgentFor(owner); - Container.get(SourceControlPreferencesService).isSourceControlConnected = () => true; + sourceControlPreferencesService = Container.get(SourceControlPreferencesService); + await sourceControlPreferencesService.setPreferences({ + connected: true, + keyGeneratorType: 'rsa', + }); }); describe('GET /sourceControl/preferences', () => { @@ -65,19 +70,11 @@ describe('GET /sourceControl/preferences', () => { }); test('refreshing key pairsshould return new rsa key', async () => { - config.set('sourceControl.defaultKeyPairType', 'rsa'); - await authOwnerAgent - .post('/source-control/generate-key-pair') - .send() - .expect(200) - .expect((res) => { - expect( - Container.get(SourceControlPreferencesService).getPreferences().keyGeneratorType, - ).toBe('rsa'); - expect(res.body.data).toHaveProperty('publicKey'); - expect(res.body.data).toHaveProperty('keyGeneratorType'); - expect(res.body.data.keyGeneratorType).toBe('rsa'); - expect(res.body.data.publicKey).toContain('ssh-rsa'); - }); + const res = await authOwnerAgent.post('/source-control/generate-key-pair').send().expect(200); + + expect(res.body.data).toHaveProperty('publicKey'); + expect(res.body.data).toHaveProperty('keyGeneratorType'); + expect(res.body.data.keyGeneratorType).toBe('rsa'); + expect(res.body.data.publicKey).toContain('ssh-rsa'); }); }); diff --git a/packages/cli/test/integration/evaluation/test-runs.api.test.ts b/packages/cli/test/integration/evaluation/test-runs.api.test.ts index 3fcc321cc9..9690d1b50e 100644 --- a/packages/cli/test/integration/evaluation/test-runs.api.test.ts +++ b/packages/cli/test/integration/evaluation/test-runs.api.test.ts @@ -1,4 +1,5 @@ import { Container } from '@n8n/di'; +import { mockInstance } from 'n8n-core/test/utils'; import type { TestDefinition } from '@/databases/entities/test-definition.ee'; import type { User } from '@/databases/entities/user'; @@ -6,6 +7,7 @@ import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee'; import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; +import { TestRunnerService } from '@/evaluation.ee/test-runner/test-runner.service.ee'; import { createUserShell } from '@test-integration/db/users'; import { createWorkflow } from '@test-integration/db/workflows'; import * as testDb from '@test-integration/test-db'; @@ -19,9 +21,11 @@ let testDefinition: TestDefinition; let otherTestDefinition: TestDefinition; let ownerShell: User; +const testRunner = mockInstance(TestRunnerService); + const testServer = utils.setupTestServer({ endpointGroups: ['workflows', 'evaluation'], - enabledFeatures: ['feat:sharing'], + enabledFeatures: ['feat:sharing', 'feat:multipleMainInstances'], }); beforeAll(async () => { @@ -57,13 +61,13 @@ describe('GET /evaluation/test-definitions/:testDefinitionId/runs', () => { expect(resp.body.data).toEqual([]); }); - test('should retrieve 404 if test definition does not exist', async () => { + test('should return 404 if test definition does not exist', async () => { const resp = await authOwnerAgent.get('/evaluation/test-definitions/123/runs'); expect(resp.statusCode).toBe(404); }); - test('should retrieve 404 if user does not have access to test definition', async () => { + test('should return 404 if user does not have access to test definition', async () => { const resp = await authOwnerAgent.get( `/evaluation/test-definitions/${otherTestDefinition.id}/runs`, ); @@ -151,7 +155,7 @@ describe('GET /evaluation/test-definitions/:testDefinitionId/runs/:id', () => { ); }); - test('should retrieve 404 if test run does not exist', async () => { + test('should return 404 if test run does not exist', async () => { const resp = await authOwnerAgent.get( `/evaluation/test-definitions/${testDefinition.id}/runs/123`, ); @@ -159,7 +163,7 @@ describe('GET /evaluation/test-definitions/:testDefinitionId/runs/:id', () => { expect(resp.statusCode).toBe(404); }); - test('should retrieve 404 if user does not have access to test definition', async () => { + test('should return 404 if user does not have access to test definition', async () => { const testRunRepository = Container.get(TestRunRepository); const testRun = await testRunRepository.createTestRun(otherTestDefinition.id); @@ -218,7 +222,7 @@ describe('DELETE /evaluation/test-definitions/:testDefinitionId/runs/:id', () => expect(testRunAfterDelete).toBeNull(); }); - test('should retrieve 404 if test run does not exist', async () => { + test('should return 404 if test run does not exist', async () => { const resp = await authOwnerAgent.delete( `/evaluation/test-definitions/${testDefinition.id}/runs/123`, ); @@ -226,7 +230,7 @@ describe('DELETE /evaluation/test-definitions/:testDefinitionId/runs/:id', () => expect(resp.statusCode).toBe(404); }); - test('should retrieve 404 if user does not have access to test definition', async () => { + test('should return 404 if user does not have access to test definition', async () => { const testRunRepository = Container.get(TestRunRepository); const testRun = await testRunRepository.createTestRun(otherTestDefinition.id); @@ -237,3 +241,46 @@ describe('DELETE /evaluation/test-definitions/:testDefinitionId/runs/:id', () => expect(resp.statusCode).toBe(404); }); }); + +describe('POST /evaluation/test-definitions/:testDefinitionId/runs/:id/cancel', () => { + test('should cancel test run', async () => { + const testRunRepository = Container.get(TestRunRepository); + const testRun = await testRunRepository.createTestRun(testDefinition.id); + + jest.spyOn(testRunRepository, 'markAsCancelled'); + + const resp = await authOwnerAgent.post( + `/evaluation/test-definitions/${testDefinition.id}/runs/${testRun.id}/cancel`, + ); + + expect(resp.statusCode).toBe(202); + expect(resp.body).toEqual({ success: true }); + + expect(testRunner.cancelTestRun).toHaveBeenCalledWith(testRun.id); + }); + + test('should return 404 if test run does not exist', async () => { + const resp = await authOwnerAgent.post( + `/evaluation/test-definitions/${testDefinition.id}/runs/123/cancel`, + ); + + expect(resp.statusCode).toBe(404); + }); + + test('should return 404 if test definition does not exist', async () => { + const resp = await authOwnerAgent.post('/evaluation/test-definitions/123/runs/123/cancel'); + + expect(resp.statusCode).toBe(404); + }); + + test('should return 404 if user does not have access to test definition', async () => { + const testRunRepository = Container.get(TestRunRepository); + const testRun = await testRunRepository.createTestRun(otherTestDefinition.id); + + const resp = await authOwnerAgent.post( + `/evaluation/test-definitions/${otherTestDefinition.id}/runs/${testRun.id}/cancel`, + ); + + expect(resp.statusCode).toBe(404); + }); +}); diff --git a/packages/cli/test/integration/project.api.test.ts b/packages/cli/test/integration/project.api.test.ts index a75aad9566..5f7c7e1b27 100644 --- a/packages/cli/test/integration/project.api.test.ts +++ b/packages/cli/test/integration/project.api.test.ts @@ -1,10 +1,10 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; import { EntityNotFoundError } from '@n8n/typeorm'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; import type { Project } from '@/databases/entities/project'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { GlobalRole } from '@/databases/entities/user'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; @@ -177,11 +177,7 @@ describe('GET /projects/my-projects', () => { // // ACT // - const resp = await testServer - .authAgentFor(testUser1) - .get('/projects/my-projects') - .query({ includeScopes: true }) - .expect(200); + const resp = await testServer.authAgentFor(testUser1).get('/projects/my-projects').expect(200); const respProjects: Array = resp.body.data; @@ -258,11 +254,7 @@ describe('GET /projects/my-projects', () => { // // ACT // - const resp = await testServer - .authAgentFor(ownerUser) - .get('/projects/my-projects') - .query({ includeScopes: true }) - .expect(200); + const resp = await testServer.authAgentFor(ownerUser).get('/projects/my-projects').expect(200); const respProjects: Array = resp.body.data; diff --git a/packages/cli/test/integration/public-api/workflows.test.ts b/packages/cli/test/integration/public-api/workflows.test.ts index 943d33bc35..7a1fa9591d 100644 --- a/packages/cli/test/integration/public-api/workflows.test.ts +++ b/packages/cli/test/integration/public-api/workflows.test.ts @@ -1,8 +1,8 @@ +import { GlobalConfig } from '@n8n/config'; import { Container } from '@n8n/di'; import type { INode } from 'n8n-workflow'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; -import config from '@/config'; import { STARTING_NODES } from '@/constants'; import type { Project } from '@/databases/entities/project'; import type { TagEntity } from '@/databases/entities/tag-entity'; @@ -36,6 +36,8 @@ let activeWorkflowManager: ActiveWorkflowManager; const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] }); const license = testServer.license; +const globalConfig = Container.get(GlobalConfig); + mockInstance(ExecutionService); beforeAll(async () => { @@ -69,6 +71,8 @@ beforeEach(async () => { authOwnerAgent = testServer.publicApiAgentFor(owner); authMemberAgent = testServer.publicApiAgentFor(member); + + globalConfig.tags.disabled = false; }); afterEach(async () => { @@ -263,8 +267,12 @@ describe('GET /workflows', () => { test('for owner, should return all workflows filtered by `projectId`', async () => { license.setQuota('quota:maxTeamProjects', -1); - const firstProject = await Container.get(ProjectService).createTeamProject('First', owner); - const secondProject = await Container.get(ProjectService).createTeamProject('Second', member); + const firstProject = await Container.get(ProjectService).createTeamProject(owner, { + name: 'First', + }); + const secondProject = await Container.get(ProjectService).createTeamProject(member, { + name: 'Second', + }); await Promise.all([ createWorkflow({ name: 'First workflow' }, firstProject), @@ -285,10 +293,9 @@ describe('GET /workflows', () => { test('for member, should return all member-accessible workflows filtered by `projectId`', async () => { license.setQuota('quota:maxTeamProjects', -1); - const otherProject = await Container.get(ProjectService).createTeamProject( - 'Other project', - member, - ); + const otherProject = await Container.get(ProjectService).createTeamProject(member, { + name: 'Other project', + }); await Promise.all([ createWorkflow({}, member), @@ -1284,8 +1291,8 @@ describe('GET /workflows/:id/tags', () => { test('should fail due to invalid API Key', testWithAPIKey('get', '/workflows/2/tags', 'abcXYZ')); - test('should fail if workflowTagsDisabled', async () => { - config.set('workflowTagsDisabled', true); + test('should fail if N8N_WORKFLOW_TAGS_DISABLED', async () => { + globalConfig.tags.disabled = true; const response = await authOwnerAgent.get('/workflows/2/tags'); @@ -1294,16 +1301,12 @@ describe('GET /workflows/:id/tags', () => { }); test('should fail due to non-existing workflow', async () => { - config.set('workflowTagsDisabled', false); - const response = await authOwnerAgent.get('/workflows/2/tags'); expect(response.statusCode).toBe(404); }); test('should return all tags of owned workflow', async () => { - config.set('workflowTagsDisabled', false); - const tags = await Promise.all([await createTag({}), await createTag({})]); const workflow = await createWorkflow({ tags }, member); @@ -1328,8 +1331,6 @@ describe('GET /workflows/:id/tags', () => { }); test('should return empty array if workflow does not have tags', async () => { - config.set('workflowTagsDisabled', false); - const workflow = await createWorkflow({}, member); const response = await authMemberAgent.get(`/workflows/${workflow.id}/tags`); @@ -1344,8 +1345,8 @@ describe('PUT /workflows/:id/tags', () => { test('should fail due to invalid API Key', testWithAPIKey('put', '/workflows/2/tags', 'abcXYZ')); - test('should fail if workflowTagsDisabled', async () => { - config.set('workflowTagsDisabled', true); + test('should fail if N8N_WORKFLOW_TAGS_DISABLED', async () => { + globalConfig.tags.disabled = true; const response = await authOwnerAgent.put('/workflows/2/tags').send([]); @@ -1354,16 +1355,12 @@ describe('PUT /workflows/:id/tags', () => { }); test('should fail due to non-existing workflow', async () => { - config.set('workflowTagsDisabled', false); - const response = await authOwnerAgent.put('/workflows/2/tags').send([]); expect(response.statusCode).toBe(404); }); test('should add the tags, workflow have not got tags previously', async () => { - config.set('workflowTagsDisabled', false); - const workflow = await createWorkflow({}, member); const tags = await Promise.all([await createTag({}), await createTag({})]); @@ -1422,8 +1419,6 @@ describe('PUT /workflows/:id/tags', () => { }); test('should add the tags, workflow have some tags previously', async () => { - config.set('workflowTagsDisabled', false); - const tags = await Promise.all([await createTag({}), await createTag({}), await createTag({})]); const oldTags = [tags[0], tags[1]]; const newTags = [tags[0], tags[2]]; @@ -1510,8 +1505,6 @@ describe('PUT /workflows/:id/tags', () => { }); test('should fail to add the tags as one does not exist, workflow should maintain previous tags', async () => { - config.set('workflowTagsDisabled', false); - const tags = await Promise.all([await createTag({}), await createTag({})]); const oldTags = [tags[0], tags[1]]; const workflow = await createWorkflow({ tags: oldTags }, member); diff --git a/packages/cli/test/integration/role.api.test.ts b/packages/cli/test/integration/role.api.test.ts index 85888347cc..d8f56604a3 100644 --- a/packages/cli/test/integration/role.api.test.ts +++ b/packages/cli/test/integration/role.api.test.ts @@ -1,7 +1,7 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { CredentialSharingRole } from '@/databases/entities/shared-credentials'; import type { WorkflowSharingRole } from '@/databases/entities/shared-workflow'; import type { GlobalRole } from '@/databases/entities/user'; diff --git a/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts index ddf3fce556..05a6b2c31f 100644 --- a/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/instance-risk-reporter.test.ts @@ -240,6 +240,7 @@ test('should not report outdated instance when up to date', async () => { test('should report security settings', async () => { Container.get(GlobalConfig).diagnostics.enabled = true; + const testAudit = await securityAuditService.run(['instance']); const section = getRiskSection( diff --git a/packages/cli/test/integration/services/project.service.test.ts b/packages/cli/test/integration/services/project.service.test.ts index bf83c6159c..b01aed3b3b 100644 --- a/packages/cli/test/integration/services/project.service.test.ts +++ b/packages/cli/test/integration/services/project.service.test.ts @@ -1,7 +1,7 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { Scope } from '@n8n/permissions'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { ProjectService } from '@/services/project.service.ee'; diff --git a/packages/cli/test/integration/shared/db/projects.ts b/packages/cli/test/integration/shared/db/projects.ts index 6ca0fcfad2..9d61c2a667 100644 --- a/packages/cli/test/integration/shared/db/projects.ts +++ b/packages/cli/test/integration/shared/db/projects.ts @@ -1,7 +1,8 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { Project } from '@/databases/entities/project'; -import type { ProjectRelation, ProjectRole } from '@/databases/entities/project-relation'; +import type { ProjectRelation } from '@/databases/entities/project-relation'; import type { User } from '@/databases/entities/user'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import { ProjectRepository } from '@/databases/repositories/project.repository'; diff --git a/packages/cli/test/integration/shared/db/users.ts b/packages/cli/test/integration/shared/db/users.ts index bb4332f9de..af0cf99820 100644 --- a/packages/cli/test/integration/shared/db/users.ts +++ b/packages/cli/test/integration/shared/db/users.ts @@ -81,7 +81,10 @@ export async function createUserWithMfaEnabled( } export const addApiKey = async (user: User) => { - return await Container.get(PublicApiKeyService).createPublicApiKeyForUser(user); + return await Container.get(PublicApiKeyService).createPublicApiKeyForUser(user, { + label: randomName(), + expiresAt: null, + }); }; export async function createOwnerWithApiKey() { diff --git a/packages/cli/test/integration/shared/utils/task-broker-test-server.ts b/packages/cli/test/integration/shared/utils/task-broker-test-server.ts index 63a96678a2..a908d5ce5d 100644 --- a/packages/cli/test/integration/shared/utils/task-broker-test-server.ts +++ b/packages/cli/test/integration/shared/utils/task-broker-test-server.ts @@ -3,10 +3,10 @@ import { Container } from '@n8n/di'; import request from 'supertest'; import type TestAgent from 'supertest/lib/agent'; -import { TaskRunnerServer } from '@/task-runners/task-runner-server'; +import { TaskBrokerServer } from '@/task-runners/task-broker/task-broker-server'; export interface TestTaskBrokerServer { - server: TaskRunnerServer; + server: TaskBrokerServer; agent: TestAgent; config: TaskRunnersConfig; } @@ -29,11 +29,11 @@ export const setupBrokerTestServer = ( runnerConfig.enabled = true; runnerConfig.port = 0; // Use any port - const taskRunnerServer = Container.get(TaskRunnerServer); - const agent = request.agent(taskRunnerServer.app); + const taskBrokerServer = Container.get(TaskBrokerServer); + const agent = request.agent(taskBrokerServer.app); return { - server: taskRunnerServer, + server: taskBrokerServer, agent, config: runnerConfig, }; diff --git a/packages/cli/test/integration/tags.api.test.ts b/packages/cli/test/integration/tags.api.test.ts index 5d9a724f78..90d764f534 100644 --- a/packages/cli/test/integration/tags.api.test.ts +++ b/packages/cli/test/integration/tags.api.test.ts @@ -8,6 +8,7 @@ import type { SuperAgentTest } from './shared/types'; import * as utils from './shared/utils/'; let authOwnerAgent: SuperAgentTest; + const testServer = utils.setupTestServer({ endpointGroups: ['tags'] }); beforeAll(async () => { @@ -22,8 +23,8 @@ beforeEach(async () => { describe('POST /tags', () => { test('should create tag', async () => { const resp = await authOwnerAgent.post('/tags').send({ name: 'test' }); - expect(resp.statusCode).toBe(200); + expect(resp.statusCode).toBe(200); const dbTag = await Container.get(TagRepository).findBy({ name: 'test' }); expect(dbTag.length === 1); }); @@ -38,4 +39,59 @@ describe('POST /tags', () => { const dbTag = await Container.get(TagRepository).findBy({ name: 'test' }); expect(dbTag.length).toBe(1); }); + + test('should delete tag', async () => { + const newTag = Container.get(TagRepository).create({ name: 'test' }); + await Container.get(TagRepository).save(newTag); + + const resp = await authOwnerAgent.delete(`/tags/${newTag.id}`); + expect(resp.status).toBe(200); + + const dbTag = await Container.get(TagRepository).findBy({ name: 'test' }); + expect(dbTag.length).toBe(0); + }); + + test('should update tag name', async () => { + const newTag = Container.get(TagRepository).create({ name: 'test' }); + await Container.get(TagRepository).save(newTag); + + const resp = await authOwnerAgent.patch(`/tags/${newTag.id}`).send({ name: 'updated' }); + expect(resp.status).toBe(200); + + const dbTag = await Container.get(TagRepository).findBy({ name: 'updated' }); + expect(dbTag.length).toBe(1); + }); + + test('should retrieve all tags', async () => { + const newTag = Container.get(TagRepository).create({ name: 'test' }); + const savedTag = await Container.get(TagRepository).save(newTag); + + const resp = await authOwnerAgent.get('/tags'); + expect(resp.status).toBe(200); + + expect(resp.body.data.length).toBe(1); + expect(resp.body.data[0]).toMatchObject({ + id: savedTag.id, + name: savedTag.name, + createdAt: savedTag.createdAt.toISOString(), + updatedAt: savedTag.updatedAt.toISOString(), + }); + }); + + test('should retrieve all tags with with usage count', async () => { + const newTag = Container.get(TagRepository).create({ name: 'test' }); + const savedTag = await Container.get(TagRepository).save(newTag); + + const resp = await authOwnerAgent.get('/tags').query({ withUsageCount: 'true' }); + expect(resp.status).toBe(200); + + expect(resp.body.data.length).toBe(1); + expect(resp.body.data[0]).toMatchObject({ + id: savedTag.id, + name: savedTag.name, + createdAt: savedTag.createdAt.toISOString(), + updatedAt: savedTag.updatedAt.toISOString(), + usageCount: 0, + }); + }); }); diff --git a/packages/cli/test/integration/task-runners/js-task-runner-execution.integration.test.ts b/packages/cli/test/integration/task-runners/js-task-runner-execution.integration.test.ts new file mode 100644 index 0000000000..a69f60d198 --- /dev/null +++ b/packages/cli/test/integration/task-runners/js-task-runner-execution.integration.test.ts @@ -0,0 +1,269 @@ +import { TaskRunnersConfig } from '@n8n/config'; +import { Container } from '@n8n/di'; +import { mock } from 'jest-mock-extended'; +import type { + IExecuteFunctions, + INode, + INodeExecutionData, + INodeParameters, + INodeTypes, + IRunExecutionData, + ITaskDataConnections, + IWorkflowExecuteAdditionalData, + WorkflowExecuteMode, +} from 'n8n-workflow'; +import { createEnvProviderState, NodeConnectionType, Workflow } from 'n8n-workflow'; + +import { LocalTaskRequester } from '@/task-runners/task-managers/local-task-requester'; +import { TaskRunnerModule } from '@/task-runners/task-runner-module'; + +/** + * Integration tests for the JS TaskRunner execution. Starts the TaskRunner + * as a child process and executes tasks on it via the broker. + */ +describe('JS TaskRunner execution on internal mode', () => { + const runnerConfig = Container.get(TaskRunnersConfig); + runnerConfig.mode = 'internal'; + runnerConfig.enabled = true; + runnerConfig.port = 45678; + + const taskRunnerModule = Container.get(TaskRunnerModule); + const taskRequester = Container.get(LocalTaskRequester); + + /** + * Sets up task data that includes a workflow with manual trigger and a + * code node with the given JS code. The input data is a single item: + * ```json + * { + * "input": "item" + * } + * ``` + */ + const newTaskData = (jsCode: string) => { + const taskSettings = { + code: jsCode, + nodeMode: 'runOnceForAllItems', + workflowMode: 'manual', + continueOnFail: false, + }; + + const codeNode: INode = { + parameters: { + jsCode, + }, + type: 'n8n-nodes-base.code', + typeVersion: 2, + position: [200, 80], + id: 'b35fd455-32e4-4d52-b840-36aa28dd1910', + name: 'Code', + }; + + const workflow = new Workflow({ + id: 'testWorkflow', + name: 'testWorkflow', + nodes: [ + { + parameters: {}, + type: 'n8n-nodes-base.manualTrigger', + typeVersion: 1, + position: [0, 0], + id: 'a39a566a-283a-433e-88bc-b3857aab706f', + name: 'ManualTrigger', + }, + codeNode, + ], + connections: { + ManualTrigger: { + main: [ + [ + { + node: 'Code', + type: NodeConnectionType.Main, + index: 0, + }, + ], + ], + }, + }, + active: true, + nodeTypes: mock(), + }); + + const inputData: INodeExecutionData[] = [ + { + json: { + input: 'item', + }, + }, + ]; + + const inputConnections: ITaskDataConnections = { + main: [inputData], + }; + + const runExecutionData: IRunExecutionData = { + startData: {}, + resultData: { + runData: { + ManualTrigger: [ + { + startTime: Date.now(), + executionTime: 0, + executionStatus: 'success', + source: [], + data: { + main: [inputData], + }, + }, + ], + }, + lastNodeExecuted: 'ManualTrigger', + }, + executionData: { + contextData: {}, + nodeExecutionStack: [], + metadata: {}, + waitingExecution: {}, + waitingExecutionSource: {}, + }, + }; + + return { + additionalData: mock(), + executeFunctions: mock(), + taskSettings, + codeNode, + workflow, + inputData, + inputConnections, + runExecutionData, + envProviderState: createEnvProviderState(), + }; + }; + + const runTaskWithCode = async (jsCode: string) => { + const { + additionalData, + taskSettings, + codeNode, + workflow, + inputData, + inputConnections, + runExecutionData, + executeFunctions, + envProviderState, + } = newTaskData(jsCode); + + return await taskRequester.startTask( + additionalData, + 'javascript', + taskSettings, + executeFunctions, + inputConnections, + codeNode, + workflow, + runExecutionData, + 0, + 0, + codeNode.name, + inputData, + mock(), + mock(), + envProviderState, + ); + }; + + describe('Basic code execution', () => { + beforeAll(async () => { + await taskRunnerModule.start(); + }); + + afterAll(async () => { + await taskRunnerModule.stop(); + }); + + it('should execute a simple JS task', async () => { + // Act + const result = await runTaskWithCode('return [{ hello: "world" }]'); + + // Assert + expect(result).toEqual({ + ok: true, + result: [{ json: { hello: 'world' } }], + }); + }); + }); + + describe('Internal and external libs', () => { + beforeAll(async () => { + process.env.NODE_FUNCTION_ALLOW_BUILTIN = 'crypto'; + process.env.NODE_FUNCTION_ALLOW_EXTERNAL = 'moment'; + await taskRunnerModule.start(); + }); + + afterAll(async () => { + await taskRunnerModule.stop(); + }); + + it('should allow importing allowed internal module', async () => { + // Act + const result = await runTaskWithCode(` + const crypto = require("crypto"); + return [{ + digest: crypto + .createHmac("sha256", Buffer.from("MySecretKey")) + .update("MESSAGE") + .digest("base64") + }] + `); + + expect(result).toEqual({ + ok: true, + result: [{ json: { digest: 'T09DMv7upNDKMD3Ht36FkwzrmWSgWpPiUNlcIX9/yaI=' } }], + }); + }); + + it('should not allow importing disallowed internal module', async () => { + // Act + const result = await runTaskWithCode(` + const fs = require("fs"); + return [{ file: fs.readFileSync("test.txt") }] + `); + + expect(result).toEqual({ + ok: false, + error: expect.objectContaining({ + message: "Cannot find module 'fs' [line 2]", + }), + }); + }); + + it('should allow importing allowed external module', async () => { + // Act + const result = await runTaskWithCode(` + const moment = require("moment"); + return [{ time: moment("1995-12-25").format("YYYY-MM-DD") }] + `); + + expect(result).toEqual({ + ok: true, + result: [{ json: { time: '1995-12-25' } }], + }); + }); + + it('should not allow importing disallowed external module', async () => { + // Act + const result = await runTaskWithCode(` + const lodash = require("lodash"); + return [{ obj: lodash.cloneDeep({}) }] + `); + + expect(result).toEqual({ + ok: false, + error: expect.objectContaining({ + message: "Cannot find module 'lodash' [line 2]", + }), + }); + }); + }); +}); diff --git a/packages/cli/test/integration/task-runners/task-broker-server.test.ts b/packages/cli/test/integration/task-runners/task-broker-server.test.ts new file mode 100644 index 0000000000..ab17d54820 --- /dev/null +++ b/packages/cli/test/integration/task-runners/task-broker-server.test.ts @@ -0,0 +1,44 @@ +import { setupBrokerTestServer } from '@test-integration/utils/task-broker-test-server'; + +describe('TaskBrokerServer', () => { + const { agent, server } = setupBrokerTestServer({ + authToken: 'token', + mode: 'external', + }); + + beforeAll(async () => { + await server.start(); + }); + + afterAll(async () => { + await server.stop(); + }); + + describe('/healthz', () => { + it('should return 200', async () => { + await agent.get('/healthz').expect(200); + }); + }); + + describe('/runners/_ws', () => { + it('should return 429 when too many requests are made', async () => { + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(429); + }); + }); + + describe('/runners/auth', () => { + it('should return 429 when too many requests are made', async () => { + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(429); + }); + }); +}); diff --git a/packages/cli/test/integration/runners/task-runner-module.external.test.ts b/packages/cli/test/integration/task-runners/task-runner-module.external.test.ts similarity index 91% rename from packages/cli/test/integration/runners/task-runner-module.external.test.ts rename to packages/cli/test/integration/task-runners/task-runner-module.external.test.ts index d111854db6..f60b53bdad 100644 --- a/packages/cli/test/integration/runners/task-runner-module.external.test.ts +++ b/packages/cli/test/integration/task-runners/task-runner-module.external.test.ts @@ -3,10 +3,10 @@ import { Container } from '@n8n/di'; import { mock } from 'jest-mock-extended'; import { MissingAuthTokenError } from '@/task-runners/errors/missing-auth-token.error'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; import { TaskRunnerModule } from '@/task-runners/task-runner-module'; import { DefaultTaskRunnerDisconnectAnalyzer } from '../../../src/task-runners/default-task-runner-disconnect-analyzer'; -import { TaskRunnerWsServer } from '../../../src/task-runners/task-runner-ws-server'; describe('TaskRunnerModule in external mode', () => { const runnerConfig = Container.get(TaskRunnersConfig); @@ -46,7 +46,7 @@ describe('TaskRunnerModule in external mode', () => { }); it('should use DefaultTaskRunnerDisconnectAnalyzer', () => { - const wsServer = Container.get(TaskRunnerWsServer); + const wsServer = Container.get(TaskBrokerWsServer); expect(wsServer.getDisconnectAnalyzer()).toBeInstanceOf(DefaultTaskRunnerDisconnectAnalyzer); }); diff --git a/packages/cli/test/integration/runners/task-runner-module.internal.test.ts b/packages/cli/test/integration/task-runners/task-runner-module.internal.test.ts similarity index 88% rename from packages/cli/test/integration/runners/task-runner-module.internal.test.ts rename to packages/cli/test/integration/task-runners/task-runner-module.internal.test.ts index 3c373e9614..15876f04ac 100644 --- a/packages/cli/test/integration/runners/task-runner-module.internal.test.ts +++ b/packages/cli/test/integration/task-runners/task-runner-module.internal.test.ts @@ -1,10 +1,10 @@ import { TaskRunnersConfig } from '@n8n/config'; import { Container } from '@n8n/di'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; import { TaskRunnerModule } from '@/task-runners/task-runner-module'; import { InternalTaskRunnerDisconnectAnalyzer } from '../../../src/task-runners/internal-task-runner-disconnect-analyzer'; -import { TaskRunnerWsServer } from '../../../src/task-runners/task-runner-ws-server'; describe('TaskRunnerModule in internal mode', () => { const runnerConfig = Container.get(TaskRunnersConfig); @@ -33,7 +33,7 @@ describe('TaskRunnerModule in internal mode', () => { }); it('should use InternalTaskRunnerDisconnectAnalyzer', () => { - const wsServer = Container.get(TaskRunnerWsServer); + const wsServer = Container.get(TaskBrokerWsServer); expect(wsServer.getDisconnectAnalyzer()).toBeInstanceOf(InternalTaskRunnerDisconnectAnalyzer); }); diff --git a/packages/cli/test/integration/runners/task-runner-process.test.ts b/packages/cli/test/integration/task-runners/task-runner-process.test.ts similarity index 93% rename from packages/cli/test/integration/runners/task-runner-process.test.ts rename to packages/cli/test/integration/task-runners/task-runner-process.test.ts index bd6dae00b0..6e5100e24a 100644 --- a/packages/cli/test/integration/runners/task-runner-process.test.ts +++ b/packages/cli/test/integration/task-runners/task-runner-process.test.ts @@ -1,9 +1,9 @@ import { Container } from '@n8n/di'; -import { TaskBroker } from '@/task-runners/task-broker.service'; +import { TaskBrokerWsServer } from '@/task-runners/task-broker/task-broker-ws-server'; +import { TaskBroker } from '@/task-runners/task-broker/task-broker.service'; import { TaskRunnerProcess } from '@/task-runners/task-runner-process'; import { TaskRunnerProcessRestartLoopDetector } from '@/task-runners/task-runner-process-restart-loop-detector'; -import { TaskRunnerWsServer } from '@/task-runners/task-runner-ws-server'; import { retryUntil } from '@test-integration/retry-until'; import { setupBrokerTestServer } from '@test-integration/utils/task-broker-test-server'; @@ -13,7 +13,7 @@ describe('TaskRunnerProcess', () => { }); const runnerProcess = Container.get(TaskRunnerProcess); const taskBroker = Container.get(TaskBroker); - const taskRunnerService = Container.get(TaskRunnerWsServer); + const taskRunnerService = Container.get(TaskBrokerWsServer); beforeAll(async () => { await taskRunnerServer.start(); diff --git a/packages/cli/test/integration/runners/task-runner-server.test.ts b/packages/cli/test/integration/task-runners/task-runner-server.test.ts similarity index 97% rename from packages/cli/test/integration/runners/task-runner-server.test.ts rename to packages/cli/test/integration/task-runners/task-runner-server.test.ts index 6088af3525..ab17d54820 100644 --- a/packages/cli/test/integration/runners/task-runner-server.test.ts +++ b/packages/cli/test/integration/task-runners/task-runner-server.test.ts @@ -1,6 +1,6 @@ import { setupBrokerTestServer } from '@test-integration/utils/task-broker-test-server'; -describe('TaskRunnerServer', () => { +describe('TaskBrokerServer', () => { const { agent, server } = setupBrokerTestServer({ authToken: 'token', mode: 'external', diff --git a/packages/cli/test/integration/workflows/workflow-sharing.service.test.ts b/packages/cli/test/integration/workflows/workflow-sharing.service.test.ts index bfd068e2a0..3f54420873 100644 --- a/packages/cli/test/integration/workflows/workflow-sharing.service.test.ts +++ b/packages/cli/test/integration/workflows/workflow-sharing.service.test.ts @@ -72,7 +72,7 @@ describe('WorkflowSharingService', () => { // // ARRANGE // - const project = await projectService.createTeamProject('Team Project', member); + const project = await projectService.createTeamProject(member, { name: 'Team Project' }); await projectService.addUser(project.id, anotherMember.id, 'project:admin'); const workflow = await createWorkflow(undefined, project); @@ -93,9 +93,9 @@ describe('WorkflowSharingService', () => { // // ARRANGE // - const project1 = await projectService.createTeamProject('Team Project 1', member); + const project1 = await projectService.createTeamProject(member, { name: 'Team Project 1' }); const workflow1 = await createWorkflow(undefined, project1); - const project2 = await projectService.createTeamProject('Team Project 2', member); + const project2 = await projectService.createTeamProject(member, { name: 'Team Project 2' }); const workflow2 = await createWorkflow(undefined, project2); await projectService.addUser(project1.id, anotherMember.id, 'project:admin'); await projectService.addUser(project2.id, anotherMember.id, 'project:viewer'); diff --git a/packages/cli/test/integration/workflows/workflow.service.test.ts b/packages/cli/test/integration/workflows/workflow.service.test.ts index 8e2c76c981..b7d9c033b4 100644 --- a/packages/cli/test/integration/workflows/workflow.service.test.ts +++ b/packages/cli/test/integration/workflows/workflow.service.test.ts @@ -40,6 +40,7 @@ beforeAll(async () => { mock(), mock(), mock(), + mock(), ); }); diff --git a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts index d376500484..b95304a583 100644 --- a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts +++ b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts @@ -1,3 +1,4 @@ +import type { ProjectRole } from '@n8n/api-types'; import { Container } from '@n8n/di'; import { ApplicationError, WorkflowActivationError, type INode } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; @@ -5,7 +6,6 @@ import { v4 as uuid } from 'uuid'; import { ActiveWorkflowManager } from '@/active-workflow-manager'; import config from '@/config'; import type { Project } from '@/databases/entities/project'; -import type { ProjectRole } from '@/databases/entities/project-relation'; import type { User } from '@/databases/entities/user'; import { ProjectRepository } from '@/databases/repositories/project.repository'; import { WorkflowHistoryRepository } from '@/databases/repositories/workflow-history.repository'; diff --git a/packages/cli/test/integration/workflows/workflows.controller.test.ts b/packages/cli/test/integration/workflows/workflows.controller.test.ts index e69e172f97..55c34f633c 100644 --- a/packages/cli/test/integration/workflows/workflows.controller.test.ts +++ b/packages/cli/test/integration/workflows/workflows.controller.test.ts @@ -854,6 +854,97 @@ describe('GET /workflows', () => { }); }); }); + + describe('sortBy', () => { + test('should fail when trying to sort by non sortable column', async () => { + await authOwnerAgent.get('/workflows').query('sortBy=nonSortableColumn:asc').expect(500); + }); + + test('should sort by createdAt column', async () => { + await createWorkflow({ name: 'First' }, owner); + await createWorkflow({ name: 'Second' }, owner); + + let response = await authOwnerAgent + .get('/workflows') + .query('sortBy=createdAt:asc') + .expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'First' }), + expect.objectContaining({ name: 'Second' }), + ]), + }); + + response = await authOwnerAgent.get('/workflows').query('sortBy=createdAt:desc').expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'Second' }), + expect.objectContaining({ name: 'First' }), + ]), + }); + }); + + test('should sort by name column', async () => { + await createWorkflow({ name: 'a' }, owner); + await createWorkflow({ name: 'b' }, owner); + + let response; + + response = await authOwnerAgent.get('/workflows').query('sortBy=name:asc').expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'a' }), + expect.objectContaining({ name: 'b' }), + ]), + }); + + response = await authOwnerAgent.get('/workflows').query('sortBy=name:desc').expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'b' }), + expect.objectContaining({ name: 'a' }), + ]), + }); + }); + + test('should sort by updatedAt column', async () => { + const futureDate = new Date(); + futureDate.setDate(futureDate.getDate() + 10); + + await createWorkflow({ name: 'First', updatedAt: futureDate }, owner); + await createWorkflow({ name: 'Second' }, owner); + + let response; + + response = await authOwnerAgent.get('/workflows').query('sortBy=updatedAt:asc').expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'Second' }), + expect.objectContaining({ name: 'First' }), + ]), + }); + + response = await authOwnerAgent.get('/workflows').query('sortBy=name:desc').expect(200); + + expect(response.body).toEqual({ + count: 2, + data: arrayContaining([ + expect.objectContaining({ name: 'First' }), + expect.objectContaining({ name: 'Second' }), + ]), + }); + }); + }); }); describe('PATCH /workflows/:workflowId', () => { diff --git a/packages/cli/test/setup-test-folder.ts b/packages/cli/test/setup-test-folder.ts index 8a58c48f86..80b9953333 100644 --- a/packages/cli/test/setup-test-folder.ts +++ b/packages/cli/test/setup-test-folder.ts @@ -20,3 +20,8 @@ writeFileSync( mode: 0o600, }, ); + +// This is needed to ensure that `process.env` overrides in tests +// are set before any of the config classes are instantiated. +// TODO: delete this after we are done migrating everything to config classes +import '@/config'; diff --git a/packages/core/.eslintrc.js b/packages/core/.eslintrc.js index 9bfae8a9eb..ceb431ce40 100644 --- a/packages/core/.eslintrc.js +++ b/packages/core/.eslintrc.js @@ -16,6 +16,7 @@ module.exports = { rules: { complexity: 'error', + 'unicorn/filename-case': ['error', { case: 'kebabCase' }], // TODO: Remove this '@typescript-eslint/ban-ts-comment': ['error', { 'ts-ignore': true }], diff --git a/packages/core/bin/copy-icons b/packages/core/bin/copy-static-files similarity index 56% rename from packages/core/bin/copy-icons rename to packages/core/bin/copy-static-files index bdcb011c25..a178a40816 100755 --- a/packages/core/bin/copy-icons +++ b/packages/core/bin/copy-static-files @@ -6,13 +6,18 @@ const { cp } = require('fs/promises'); const { packageDir } = require('./common'); const limiter = pLimit(20); -const icons = glob.sync('{nodes,credentials}/**/*.{png,svg}', { cwd: packageDir }); +const staticFiles = glob.sync( + ['{nodes,credentials}/**/*.{png,svg}', 'nodes/**/__schema__/**/*.json'], + { + cwd: packageDir, + }, +); (async () => { await Promise.all( - icons.map((icon) => + staticFiles.map((path) => limiter(() => { - return cp(icon, `dist/${icon}`, { recursive: true }); + return cp(path, `dist/${path}`, { recursive: true }); }), ), ); diff --git a/packages/core/bin/generate-metadata b/packages/core/bin/generate-metadata index 18dbca687a..b4001ccd84 100755 --- a/packages/core/bin/generate-metadata +++ b/packages/core/bin/generate-metadata @@ -1,7 +1,7 @@ #!/usr/bin/env node const { LoggerProxy } = require('n8n-workflow'); -const { PackageDirectoryLoader } = require('../dist/DirectoryLoader'); +const { PackageDirectoryLoader } = require('../dist/nodes-loader/package-directory-loader'); const { packageDir, writeJSON } = require('./common'); LoggerProxy.init(console); diff --git a/packages/core/package.json b/packages/core/package.json index 8773d134ea..b7bf18088c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,11 +1,11 @@ { "name": "n8n-core", - "version": "1.74.0", + "version": "1.78.0", "description": "Core functionality of n8n", "main": "dist/index", "types": "dist/index.d.ts", "bin": { - "n8n-copy-icons": "./bin/copy-icons", + "n8n-copy-static-files": "./bin/copy-static-files", "n8n-generate-translations": "./bin/generate-translations", "n8n-generate-metadata": "./bin/generate-metadata" }, diff --git a/packages/core/src/CreateNodeAsTool.ts b/packages/core/src/CreateNodeAsTool.ts deleted file mode 100644 index da34b377df..0000000000 --- a/packages/core/src/CreateNodeAsTool.ts +++ /dev/null @@ -1,419 +0,0 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import type { IDataObject, INode, INodeType } from 'n8n-workflow'; -import { jsonParse, NodeOperationError } from 'n8n-workflow'; -import { z } from 'zod'; - -type AllowedTypes = 'string' | 'number' | 'boolean' | 'json'; -interface FromAIArgument { - key: string; - description?: string; - type?: AllowedTypes; - defaultValue?: string | number | boolean | Record; -} - -type ParserOptions = { - node: INode; - nodeType: INodeType; - handleToolInvocation: (toolArgs: IDataObject) => Promise; -}; - -// This file is temporarily duplicated in `packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/v2/utils/FromAIParser.ts` -// Please apply any changes in both files - -/** - * AIParametersParser - * - * This class encapsulates the logic for parsing node parameters, extracting $fromAI calls, - * generating Zod schemas, and creating LangChain tools. - */ -class AIParametersParser { - /** - * Constructs an instance of AIParametersParser. - */ - constructor(private readonly options: ParserOptions) {} - - /** - * Generates a Zod schema based on the provided FromAIArgument placeholder. - * @param placeholder The FromAIArgument object containing key, type, description, and defaultValue. - * @returns A Zod schema corresponding to the placeholder's type and constraints. - */ - private generateZodSchema(placeholder: FromAIArgument): z.ZodTypeAny { - let schema: z.ZodTypeAny; - - switch (placeholder.type?.toLowerCase()) { - case 'string': - schema = z.string(); - break; - case 'number': - schema = z.number(); - break; - case 'boolean': - schema = z.boolean(); - break; - case 'json': - schema = z.record(z.any()); - break; - default: - schema = z.string(); - } - - if (placeholder.description) { - schema = schema.describe(`${schema.description ?? ''} ${placeholder.description}`.trim()); - } - - if (placeholder.defaultValue !== undefined) { - schema = schema.default(placeholder.defaultValue); - } - - return schema; - } - - /** - * Recursively traverses the nodeParameters object to find all $fromAI calls. - * @param payload The current object or value being traversed. - * @param collectedArgs The array collecting FromAIArgument objects. - */ - private traverseNodeParameters(payload: unknown, collectedArgs: FromAIArgument[]) { - if (typeof payload === 'string') { - const fromAICalls = this.extractFromAICalls(payload); - fromAICalls.forEach((call) => collectedArgs.push(call)); - } else if (Array.isArray(payload)) { - payload.forEach((item: unknown) => this.traverseNodeParameters(item, collectedArgs)); - } else if (typeof payload === 'object' && payload !== null) { - Object.values(payload).forEach((value) => this.traverseNodeParameters(value, collectedArgs)); - } - } - - /** - * Extracts all $fromAI calls from a given string - * @param str The string to search for $fromAI calls. - * @returns An array of FromAIArgument objects. - * - * This method uses a regular expression to find the start of each $fromAI function call - * in the input string. It then employs a character-by-character parsing approach to - * accurately extract the arguments of each call, handling nested parentheses and quoted strings. - * - * The parsing process: - * 1. Finds the starting position of a $fromAI call using regex. - * 2. Iterates through characters, keeping track of parentheses depth and quote status. - * 3. Handles escaped characters within quotes to avoid premature quote closing. - * 4. Builds the argument string until the matching closing parenthesis is found. - * 5. Parses the extracted argument string into a FromAIArgument object. - * 6. Repeats the process for all $fromAI calls in the input string. - * - */ - private extractFromAICalls(str: string): FromAIArgument[] { - const args: FromAIArgument[] = []; - // Regular expression to match the start of a $fromAI function call - const pattern = /\$fromAI\s*\(\s*/gi; - let match: RegExpExecArray | null; - - while ((match = pattern.exec(str)) !== null) { - const startIndex = match.index + match[0].length; - let current = startIndex; - let inQuotes = false; - let quoteChar = ''; - let parenthesesCount = 1; - let argsString = ''; - - // Parse the arguments string, handling nested parentheses and quotes - while (current < str.length && parenthesesCount > 0) { - const char = str[current]; - - if (inQuotes) { - // Handle characters inside quotes, including escaped characters - if (char === '\\' && current + 1 < str.length) { - argsString += char + str[current + 1]; - current += 2; - continue; - } - - if (char === quoteChar) { - inQuotes = false; - quoteChar = ''; - } - argsString += char; - } else { - // Handle characters outside quotes - if (['"', "'", '`'].includes(char)) { - inQuotes = true; - quoteChar = char; - } else if (char === '(') { - parenthesesCount++; - } else if (char === ')') { - parenthesesCount--; - } - - // Only add characters if we're still inside the main parentheses - if (parenthesesCount > 0 || char !== ')') { - argsString += char; - } - } - - current++; - } - - // If parentheses are balanced, parse the arguments - if (parenthesesCount === 0) { - try { - const parsedArgs = this.parseArguments(argsString); - args.push(parsedArgs); - } catch (error) { - // If parsing fails, throw an ApplicationError with details - throw new NodeOperationError( - this.options.node, - `Failed to parse $fromAI arguments: ${argsString}: ${error}`, - ); - } - } else { - // Log an error if parentheses are unbalanced - throw new NodeOperationError( - this.options.node, - `Unbalanced parentheses while parsing $fromAI call: ${str.slice(startIndex)}`, - ); - } - } - - return args; - } - - /** - * Parses the arguments of a single $fromAI function call. - * @param argsString The string containing the function arguments. - * @returns A FromAIArgument object. - */ - private parseArguments(argsString: string): FromAIArgument { - // Split arguments by commas not inside quotes - const args: string[] = []; - let currentArg = ''; - let inQuotes = false; - let quoteChar = ''; - let escapeNext = false; - - for (let i = 0; i < argsString.length; i++) { - const char = argsString[i]; - - if (escapeNext) { - currentArg += char; - escapeNext = false; - continue; - } - - if (char === '\\') { - escapeNext = true; - continue; - } - - if (['"', "'", '`'].includes(char)) { - if (!inQuotes) { - inQuotes = true; - quoteChar = char; - currentArg += char; - } else if (char === quoteChar) { - inQuotes = false; - quoteChar = ''; - currentArg += char; - } else { - currentArg += char; - } - continue; - } - - if (char === ',' && !inQuotes) { - args.push(currentArg.trim()); - currentArg = ''; - continue; - } - - currentArg += char; - } - - if (currentArg) { - args.push(currentArg.trim()); - } - - // Remove surrounding quotes if present - const cleanArgs = args.map((arg) => { - const trimmed = arg.trim(); - if ( - (trimmed.startsWith("'") && trimmed.endsWith("'")) || - (trimmed.startsWith('`') && trimmed.endsWith('`')) || - (trimmed.startsWith('"') && trimmed.endsWith('"')) - ) { - return trimmed - .slice(1, -1) - .replace(/\\'/g, "'") - .replace(/\\`/g, '`') - .replace(/\\"/g, '"') - .replace(/\\\\/g, '\\'); - } - return trimmed; - }); - - const type = cleanArgs?.[2] || 'string'; - - if (!['string', 'number', 'boolean', 'json'].includes(type.toLowerCase())) { - throw new NodeOperationError(this.options.node, `Invalid type: ${type}`); - } - - return { - key: cleanArgs[0] || '', - description: cleanArgs[1], - type: (cleanArgs?.[2] ?? 'string') as AllowedTypes, - defaultValue: this.parseDefaultValue(cleanArgs[3]), - }; - } - - /** - * Parses the default value, preserving its original type. - * @param value The default value as a string. - * @returns The parsed default value in its appropriate type. - */ - private parseDefaultValue( - value: string | undefined, - ): string | number | boolean | Record | undefined { - if (value === undefined || value === '') return undefined; - const lowerValue = value.toLowerCase(); - if (lowerValue === 'true') return true; - if (lowerValue === 'false') return false; - if (!isNaN(Number(value))) return Number(value); - try { - return jsonParse(value); - } catch { - return value; - } - } - - /** - * Retrieves and validates the Zod schema for the tool. - * - * This method: - * 1. Collects all $fromAI arguments from node parameters - * 2. Validates parameter keys against naming rules - * 3. Checks for duplicate keys and ensures consistency - * 4. Generates a Zod schema from the validated arguments - * - * @throws {NodeOperationError} When parameter keys are invalid or when duplicate keys have inconsistent definitions - * @returns {z.ZodObject} A Zod schema object representing the structure and validation rules for the node parameters - */ - private getSchema() { - const { node } = this.options; - const collectedArguments: FromAIArgument[] = []; - this.traverseNodeParameters(node.parameters, collectedArguments); - - // Validate each collected argument - const nameValidationRegex = /^[a-zA-Z0-9_-]{1,64}$/; - const keyMap = new Map(); - for (const argument of collectedArguments) { - if (argument.key.length === 0 || !nameValidationRegex.test(argument.key)) { - const isEmptyError = 'You must specify a key when using $fromAI()'; - const isInvalidError = `Parameter key \`${argument.key}\` is invalid`; - const error = new Error(argument.key.length === 0 ? isEmptyError : isInvalidError); - throw new NodeOperationError(node, error, { - description: - 'Invalid parameter key, must be between 1 and 64 characters long and only contain letters, numbers, underscores, and hyphens', - }); - } - - if (keyMap.has(argument.key)) { - // If the key already exists in the Map - const existingArg = keyMap.get(argument.key)!; - - // Check if the existing argument has the same description and type - if ( - existingArg.description !== argument.description || - existingArg.type !== argument.type - ) { - // If not, throw an error for inconsistent duplicate keys - throw new NodeOperationError( - node, - `Duplicate key '${argument.key}' found with different description or type`, - { - description: - 'Ensure all $fromAI() calls with the same key have consistent descriptions and types', - }, - ); - } - // If the duplicate key has consistent description and type, it's allowed (no action needed) - } else { - // If the key doesn't exist in the Map, add it - keyMap.set(argument.key, argument); - } - } - - // Remove duplicate keys, latest occurrence takes precedence - const uniqueArgsMap = collectedArguments.reduce((map, arg) => { - map.set(arg.key, arg); - return map; - }, new Map()); - - const uniqueArguments = Array.from(uniqueArgsMap.values()); - - // Generate Zod schema from unique arguments - const schemaObj = uniqueArguments.reduce((acc: Record, placeholder) => { - acc[placeholder.key] = this.generateZodSchema(placeholder); - return acc; - }, {}); - - return z.object(schemaObj).required(); - } - - /** - * Generates a description for a node based on the provided parameters. - * @param node The node type. - * @param nodeParameters The parameters of the node. - * @returns A string description for the node. - */ - private getDescription(): string { - const { node, nodeType } = this.options; - const manualDescription = node.parameters.toolDescription as string; - - if (node.parameters.descriptionType === 'auto') { - const resource = node.parameters.resource as string; - const operation = node.parameters.operation as string; - let description = nodeType.description.description; - if (resource) { - description += `\n Resource: ${resource}`; - } - if (operation) { - description += `\n Operation: ${operation}`; - } - return description.trim(); - } - if (node.parameters.descriptionType === 'manual') { - return manualDescription ?? nodeType.description.description; - } - - return nodeType.description.description; - } - - /** - * Creates a DynamicStructuredTool from a node. - * @returns A DynamicStructuredTool instance. - */ - createTool(): DynamicStructuredTool { - const { node, nodeType } = this.options; - const schema = this.getSchema(); - const description = this.getDescription(); - const nodeName = node.name.replace(/ /g, '_'); - const name = nodeName || nodeType.description.name; - - return new DynamicStructuredTool({ - name, - description, - schema, - func: async (toolArgs: z.infer) => - await this.options.handleToolInvocation(toolArgs), - }); - } -} - -/** - * Converts node into LangChain tool by analyzing node parameters, - * identifying placeholders using the $fromAI function, and generating a Zod schema. It then creates - * a DynamicStructuredTool that can be used in LangChain workflows. - */ -export function createNodeAsTool(options: ParserOptions) { - const parser = new AIParametersParser(options); - return { response: parser.createTool() }; -} diff --git a/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts deleted file mode 100644 index 959ab78845..0000000000 --- a/packages/core/src/PartialExecutionUtils/__tests__/cleanRunData.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { IRunData } from 'n8n-workflow'; - -import { createNodeData, toITaskData } from './helpers'; -import { cleanRunData } from '../cleanRunData'; -import { DirectedGraph } from '../DirectedGraph'; - -describe('cleanRunData', () => { - // ┌─────┐ ┌─────┐ ┌─────┐ - // │node1├───►│node2├──►│node3│ - // └─────┘ └─────┘ └─────┘ - test('deletes all run data of all children and the node being passed in', () => { - // ARRANGE - const node1 = createNodeData({ name: 'Node1' }); - const node2 = createNodeData({ name: 'Node2' }); - const node3 = createNodeData({ name: 'Node3' }); - const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); - const runData: IRunData = { - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - [node2.name]: [toITaskData([{ data: { value: 2 } }])], - [node3.name]: [toITaskData([{ data: { value: 3 } }])], - }; - - // ACT - const newRunData = cleanRunData(runData, graph, new Set([node1])); - - // ASSERT - expect(newRunData).toEqual({}); - }); - - // ┌─────┐ ┌─────┐ ┌─────┐ - // │node1├───►│node2├──►│node3│ - // └─────┘ └─────┘ └─────┘ - test('retains the run data of parent nodes of the node being passed in', () => { - // ARRANGE - const node1 = createNodeData({ name: 'Node1' }); - const node2 = createNodeData({ name: 'Node2' }); - const node3 = createNodeData({ name: 'Node3' }); - const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); - const runData: IRunData = { - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - [node2.name]: [toITaskData([{ data: { value: 2 } }])], - [node3.name]: [toITaskData([{ data: { value: 3 } }])], - }; - - // ACT - const newRunData = cleanRunData(runData, graph, new Set([node2])); - - // ASSERT - expect(newRunData).toEqual({ [node1.name]: runData[node1.name] }); - }); - - // ┌─────┐ ┌─────┐ ┌─────┐ - // ┌─►│node1├───►│node2├──►│node3├─┐ - // │ └─────┘ └─────┘ └─────┘ │ - // │ │ - // └───────────────────────────────┘ - test('terminates when finding a cycle', () => { - // ARRANGE - const node1 = createNodeData({ name: 'Node1' }); - const node2 = createNodeData({ name: 'Node2' }); - const node3 = createNodeData({ name: 'Node3' }); - const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections( - { from: node1, to: node2 }, - { from: node2, to: node3 }, - { from: node3, to: node1 }, - ); - - const runData: IRunData = { - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - [node2.name]: [toITaskData([{ data: { value: 2 } }])], - [node3.name]: [toITaskData([{ data: { value: 3 } }])], - }; - - // ACT - const newRunData = cleanRunData(runData, graph, new Set([node2])); - - // ASSERT - // TODO: Find out if this is a desirable result in milestone 2 - expect(newRunData).toEqual({}); - }); - - // ┌─────┐ ┌─────┐ - // │node1├───►│node2│ - // └─────┘ └─────┘ - test('removes run data of nodes that are not in the subgraph', () => { - // ARRANGE - const node1 = createNodeData({ name: 'Node1' }); - const node2 = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph() - .addNodes(node1, node2) - .addConnections({ from: node1, to: node2 }); - // not part of the graph - const node3 = createNodeData({ name: 'Node3' }); - const runData: IRunData = { - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - [node2.name]: [toITaskData([{ data: { value: 2 } }])], - [node3.name]: [toITaskData([{ data: { value: 3 } }])], - }; - - // ACT - const newRunData = cleanRunData(runData, graph, new Set([node2])); - - // ASSERT - expect(newRunData).toEqual({ - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - }); - }); -}); diff --git a/packages/core/src/PartialExecutionUtils/index.ts b/packages/core/src/PartialExecutionUtils/index.ts deleted file mode 100644 index f85202cfa1..0000000000 --- a/packages/core/src/PartialExecutionUtils/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { DirectedGraph } from './DirectedGraph'; -export { findTriggerForPartialExecution } from './findTriggerForPartialExecution'; -export { findStartNodes } from './findStartNodes'; -export { findSubgraph } from './findSubgraph'; -export { recreateNodeExecutionStack } from './recreateNodeExecutionStack'; -export { cleanRunData } from './cleanRunData'; -export { handleCycles } from './handleCycles'; -export { filterDisabledNodes } from './filterDisabledNodes'; diff --git a/packages/core/src/__tests__/credentials.test.ts b/packages/core/src/__tests__/credentials.test.ts new file mode 100644 index 0000000000..09b7586daf --- /dev/null +++ b/packages/core/src/__tests__/credentials.test.ts @@ -0,0 +1,121 @@ +import { Container } from '@n8n/di'; +import { mock } from 'jest-mock-extended'; +import type { CredentialInformation } from 'n8n-workflow'; +import { AssertionError } from 'node:assert'; + +import { CREDENTIAL_ERRORS } from '@/constants'; +import { Cipher } from '@/encryption/cipher'; +import type { InstanceSettings } from '@/instance-settings'; + +import { Credentials } from '../credentials'; + +describe('Credentials', () => { + const nodeCredentials = { id: '123', name: 'Test Credential' }; + const credentialType = 'testApi'; + + const cipher = new Cipher(mock({ encryptionKey: 'password' })); + Container.set(Cipher, cipher); + + const setDataKey = (credentials: Credentials, key: string, data: CredentialInformation) => { + let fullData; + try { + fullData = credentials.getData(); + } catch (e) { + fullData = {}; + } + fullData[key] = data; + return credentials.setData(fullData); + }; + + describe('without nodeType set', () => { + test('should be able to set and read key data without initial data set', () => { + const credentials = new Credentials(nodeCredentials, credentialType); + + const key = 'key1'; + const newData = 1234; + + setDataKey(credentials, key, newData); + + expect(credentials.getData()[key]).toEqual(newData); + }); + + test('should be able to set and read key data with initial data set', () => { + const key = 'key2'; + + // Saved under "key1" + const initialData = 4321; + const initialDataEncoded = 'U2FsdGVkX1+0baznXt+Ag/ub8A2kHLyoLxn/rR9h4XQ='; + + const credentials = new Credentials(nodeCredentials, credentialType, initialDataEncoded); + + const newData = 1234; + + // Set and read new data + setDataKey(credentials, key, newData); + expect(credentials.getData()[key]).toEqual(newData); + + // Read the data which got provided encrypted on init + expect(credentials.getData().key1).toEqual(initialData); + }); + }); + + describe('getData', () => { + test('should throw an error when data is missing', () => { + const credentials = new Credentials(nodeCredentials, credentialType); + credentials.data = undefined; + + expect(() => credentials.getData()).toThrow(CREDENTIAL_ERRORS.NO_DATA); + }); + + test('should throw an error when decryption fails', () => { + const credentials = new Credentials(nodeCredentials, credentialType); + credentials.data = '{"key": "already-decrypted-credentials-data" }'; + + expect(() => credentials.getData()).toThrow(CREDENTIAL_ERRORS.DECRYPTION_FAILED); + + try { + credentials.getData(); + } catch (error) { + expect(error.constructor.name).toBe('CredentialDataError'); + expect(error.extra).toEqual({ ...nodeCredentials, type: credentialType }); + expect((error.cause.code as string).startsWith('ERR_OSSL_')).toBe(true); + } + }); + + test('should throw an error when JSON parsing fails', () => { + const credentials = new Credentials(nodeCredentials, credentialType); + credentials.data = cipher.encrypt('invalid-json-string'); + + expect(() => credentials.getData()).toThrow(CREDENTIAL_ERRORS.INVALID_JSON); + + try { + credentials.getData(); + } catch (error) { + expect(error.constructor.name).toBe('CredentialDataError'); + expect(error.extra).toEqual({ ...nodeCredentials, type: credentialType }); + expect(error.cause).toBeInstanceOf(SyntaxError); + expect(error.cause.message).toMatch('Unexpected token '); + } + }); + + test('should successfully decrypt and parse valid JSON credentials', () => { + const credentials = new Credentials(nodeCredentials, credentialType); + credentials.setData({ username: 'testuser', password: 'testpass' }); + + const decryptedData = credentials.getData(); + expect(decryptedData.username).toBe('testuser'); + expect(decryptedData.password).toBe('testpass'); + }); + }); + + describe('setData', () => { + test.each<{}>([[123], [null], [undefined]])( + 'should throw an AssertionError when data is %s', + (data) => { + const credentials = new Credentials<{}>(nodeCredentials, credentialType); + + expect(() => credentials.setData(data)).toThrow(AssertionError); + }, + ); + }); +}); diff --git a/packages/core/src/__tests__/node-execute-functions.test.ts b/packages/core/src/__tests__/node-execute-functions.test.ts new file mode 100644 index 0000000000..96306ac90b --- /dev/null +++ b/packages/core/src/__tests__/node-execute-functions.test.ts @@ -0,0 +1,314 @@ +import { mock } from 'jest-mock-extended'; +import type { + IHttpRequestMethods, + IHttpRequestOptions, + INode, + IRequestOptions, + IWorkflowExecuteAdditionalData, + Workflow, +} from 'n8n-workflow'; +import nock from 'nock'; + +import type { ExecutionLifecycleHooks } from '@/execution-engine/execution-lifecycle-hooks'; +import { + copyInputItems, + invokeAxios, + proxyRequestToAxios, + removeEmptyBody, +} from '@/node-execute-functions'; + +describe('NodeExecuteFunctions', () => { + describe('proxyRequestToAxios', () => { + const baseUrl = 'http://example.de'; + const workflow = mock(); + const hooks = mock(); + const additionalData = mock({ hooks }); + const node = mock(); + + beforeEach(() => { + hooks.runHook.mockClear(); + }); + + test('should rethrow an error with `status` property', async () => { + nock(baseUrl).get('/test').reply(400); + + try { + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + } catch (error) { + expect(error.status).toEqual(400); + } + }); + + test('should not throw if the response status is 200', async () => { + nock(baseUrl).get('/test').reply(200); + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + test('should throw if the response status is 403', async () => { + const headers = { 'content-type': 'text/plain' }; + nock(baseUrl).get('/test').reply(403, 'Forbidden', headers); + try { + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + } catch (error) { + expect(error.statusCode).toEqual(403); + expect(error.request).toBeUndefined(); + expect(error.response).toMatchObject({ headers, status: 403 }); + expect(error.options).toMatchObject({ + headers: { Accept: '*/*' }, + method: 'get', + url: 'http://example.de/test', + }); + expect(error.config).toBeUndefined(); + expect(error.message).toEqual('403 - "Forbidden"'); + } + expect(hooks.runHook).not.toHaveBeenCalled(); + }); + + test('should not throw if the response status is 404, but `simple` option is set to `false`', async () => { + nock(baseUrl).get('/test').reply(404, 'Not Found'); + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/test`, + simple: false, + }); + + expect(response).toEqual('Not Found'); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + test('should return full response when `resolveWithFullResponse` is set to true', async () => { + nock(baseUrl).get('/test').reply(404, 'Not Found'); + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/test`, + resolveWithFullResponse: true, + simple: false, + }); + + expect(response).toMatchObject({ + body: 'Not Found', + headers: {}, + statusCode: 404, + statusMessage: 'Not Found', + }); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + describe('redirects', () => { + test('should forward authorization header', async () => { + nock(baseUrl).get('/redirect').reply(301, '', { Location: 'https://otherdomain.com/test' }); + nock('https://otherdomain.com') + .get('/test') + .reply(200, function () { + return this.req.headers; + }); + + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + auth: { + username: 'testuser', + password: 'testpassword', + }, + headers: { + 'X-Other-Header': 'otherHeaderContent', + }, + resolveWithFullResponse: true, + }); + + expect(response.statusCode).toBe(200); + const forwardedHeaders = JSON.parse(response.body); + expect(forwardedHeaders.authorization).toBe('Basic dGVzdHVzZXI6dGVzdHBhc3N3b3Jk'); + expect(forwardedHeaders['x-other-header']).toBe('otherHeaderContent'); + }); + + test('should follow redirects by default', async () => { + nock(baseUrl) + .get('/redirect') + .reply(301, '', { Location: `${baseUrl}/test` }); + nock(baseUrl).get('/test').reply(200, 'Redirected'); + + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + resolveWithFullResponse: true, + }); + + expect(response).toMatchObject({ + body: 'Redirected', + headers: {}, + statusCode: 200, + }); + }); + + test('should not follow redirects when configured', async () => { + nock(baseUrl) + .get('/redirect') + .reply(301, '', { Location: `${baseUrl}/test` }); + nock(baseUrl).get('/test').reply(200, 'Redirected'); + + await expect( + proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + resolveWithFullResponse: true, + followRedirect: false, + }), + ).rejects.toThrowError(expect.objectContaining({ statusCode: 301 })); + }); + }); + }); + + describe('invokeAxios', () => { + const baseUrl = 'http://example.de'; + + beforeEach(() => { + nock.cleanAll(); + jest.clearAllMocks(); + }); + + it('should throw error for non-401 status codes', async () => { + nock(baseUrl).get('/test').reply(500, {}); + + await expect(invokeAxios({ url: `${baseUrl}/test` })).rejects.toThrow( + 'Request failed with status code 500', + ); + }); + + it('should throw error on 401 without digest auth challenge', async () => { + nock(baseUrl).get('/test').reply(401, {}); + + await expect( + invokeAxios( + { + url: `${baseUrl}/test`, + }, + { sendImmediately: false }, + ), + ).rejects.toThrow('Request failed with status code 401'); + }); + + it('should make successful requests', async () => { + nock(baseUrl).get('/test').reply(200, { success: true }); + + const response = await invokeAxios({ + url: `${baseUrl}/test`, + }); + + expect(response.status).toBe(200); + expect(response.data).toEqual({ success: true }); + }); + + it('should handle digest auth when receiving 401 with nonce', async () => { + nock(baseUrl) + .get('/test') + .matchHeader('authorization', 'Basic dXNlcjpwYXNz') + .once() + .reply(401, {}, { 'www-authenticate': 'Digest realm="test", nonce="abc123", qop="auth"' }); + + nock(baseUrl) + .get('/test') + .matchHeader( + 'authorization', + /^Digest username="user",realm="test",nonce="abc123",uri="\/test",qop="auth",algorithm="MD5",response="[0-9a-f]{32}"/, + ) + .reply(200, { success: true }); + + const response = await invokeAxios( + { + url: `${baseUrl}/test`, + auth: { + username: 'user', + password: 'pass', + }, + }, + { sendImmediately: false }, + ); + + expect(response.status).toBe(200); + expect(response.data).toEqual({ success: true }); + }); + }); + + describe('copyInputItems', () => { + it('should pick only selected properties', () => { + const output = copyInputItems( + [ + { + json: { + a: 1, + b: true, + c: {}, + }, + }, + ], + ['a'], + ); + expect(output).toEqual([{ a: 1 }]); + }); + + it('should convert undefined to null', () => { + const output = copyInputItems( + [ + { + json: { + a: undefined, + }, + }, + ], + ['a'], + ); + expect(output).toEqual([{ a: null }]); + }); + + it('should clone objects', () => { + const input = { + a: { b: 5 }, + }; + const output = copyInputItems( + [ + { + json: input, + }, + ], + ['a'], + ); + expect(output[0].a).toEqual(input.a); + expect(output[0].a === input.a).toEqual(false); + }); + }); + + describe('removeEmptyBody', () => { + test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( + 'Should remove empty body for %s', + async (method) => { + const requestOptions = { + method, + body: {}, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual(undefined); + }, + ); + + test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( + 'Should not remove non-empty body for %s', + async (method) => { + const requestOptions = { + method, + body: { test: true }, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual({ test: true }); + }, + ); + + test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'Should not remove empty body for %s', + async (method) => { + const requestOptions = { + method, + body: {}, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual({}); + }, + ); + }); +}); diff --git a/packages/core/test/FileSystem.manager.test.ts b/packages/core/src/binary-data/__tests__/file-system.manager.test.ts similarity index 96% rename from packages/core/test/FileSystem.manager.test.ts rename to packages/core/src/binary-data/__tests__/file-system.manager.test.ts index edb6bd5e77..fae24801f5 100644 --- a/packages/core/test/FileSystem.manager.test.ts +++ b/packages/core/src/binary-data/__tests__/file-system.manager.test.ts @@ -3,10 +3,9 @@ import fsp from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { FileSystemManager } from '@/BinaryData/FileSystem.manager'; -import { isStream } from '@/ObjectStore/utils'; - -import { toFileId, toStream } from './utils'; +import { FileSystemManager } from '@/binary-data/file-system.manager'; +import { isStream } from '@/binary-data/object-store/utils'; +import { toFileId, toStream } from '@test/utils'; jest.mock('fs'); jest.mock('fs/promises'); diff --git a/packages/core/test/ObjectStore.manager.test.ts b/packages/core/src/binary-data/__tests__/object-store.manager.test.ts similarity index 91% rename from packages/core/test/ObjectStore.manager.test.ts rename to packages/core/src/binary-data/__tests__/object-store.manager.test.ts index f01e170213..9ca99e8d7b 100644 --- a/packages/core/test/ObjectStore.manager.test.ts +++ b/packages/core/src/binary-data/__tests__/object-store.manager.test.ts @@ -1,12 +1,11 @@ import { mock } from 'jest-mock-extended'; import fs from 'node:fs/promises'; -import { ObjectStoreManager } from '@/BinaryData/ObjectStore.manager'; -import { ObjectStoreService } from '@/ObjectStore/ObjectStore.service.ee'; -import type { MetadataResponseHeaders } from '@/ObjectStore/types'; -import { isStream } from '@/ObjectStore/utils'; - -import { mockInstance, toFileId, toStream } from './utils'; +import { ObjectStoreService } from '@/binary-data/object-store/object-store.service.ee'; +import type { MetadataResponseHeaders } from '@/binary-data/object-store/types'; +import { isStream } from '@/binary-data/object-store/utils'; +import { ObjectStoreManager } from '@/binary-data/object-store.manager'; +import { mockInstance, toFileId, toStream } from '@test/utils'; jest.mock('fs/promises'); diff --git a/packages/core/test/BinaryData/utils.test.ts b/packages/core/src/binary-data/__tests__/utils.test.ts similarity index 95% rename from packages/core/test/BinaryData/utils.test.ts rename to packages/core/src/binary-data/__tests__/utils.test.ts index 50a7f165df..329345262f 100644 --- a/packages/core/test/BinaryData/utils.test.ts +++ b/packages/core/src/binary-data/__tests__/utils.test.ts @@ -1,7 +1,7 @@ import { Readable } from 'node:stream'; import { createGunzip } from 'node:zlib'; -import { binaryToBuffer } from '@/BinaryData/utils'; +import { binaryToBuffer } from '@/binary-data/utils'; describe('BinaryData/utils', () => { describe('binaryToBuffer', () => { diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/binary-data/binary-data.service.ts similarity index 96% rename from packages/core/src/BinaryData/BinaryData.service.ts rename to packages/core/src/binary-data/binary-data.service.ts index 5876ae075e..742ceb500f 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/binary-data/binary-data.service.ts @@ -22,7 +22,7 @@ export class BinaryDataService { this.mode = config.mode === 'filesystem' ? 'filesystem-v2' : config.mode; if (config.availableModes.includes('filesystem')) { - const { FileSystemManager } = await import('./FileSystem.manager'); + const { FileSystemManager } = await import('./file-system.manager'); this.managers.filesystem = new FileSystemManager(config.localStoragePath); this.managers['filesystem-v2'] = this.managers.filesystem; @@ -31,8 +31,8 @@ export class BinaryDataService { } if (config.availableModes.includes('s3')) { - const { ObjectStoreManager } = await import('./ObjectStore.manager'); - const { ObjectStoreService } = await import('../ObjectStore/ObjectStore.service.ee'); + const { ObjectStoreManager } = await import('./object-store.manager'); + const { ObjectStoreService } = await import('./object-store/object-store.service.ee'); this.managers.s3 = new ObjectStoreManager(Container.get(ObjectStoreService)); diff --git a/packages/core/src/BinaryData/FileSystem.manager.ts b/packages/core/src/binary-data/file-system.manager.ts similarity index 100% rename from packages/core/src/BinaryData/FileSystem.manager.ts rename to packages/core/src/binary-data/file-system.manager.ts diff --git a/packages/core/src/binary-data/index.ts b/packages/core/src/binary-data/index.ts new file mode 100644 index 0000000000..ef82dee30f --- /dev/null +++ b/packages/core/src/binary-data/index.ts @@ -0,0 +1,4 @@ +export * from './binary-data.service'; +export * from './types'; +export { ObjectStoreService } from './object-store/object-store.service.ee'; +export { isStoredMode as isValidNonDefaultMode } from './utils'; diff --git a/packages/core/src/BinaryData/ObjectStore.manager.ts b/packages/core/src/binary-data/object-store.manager.ts similarity index 97% rename from packages/core/src/BinaryData/ObjectStore.manager.ts rename to packages/core/src/binary-data/object-store.manager.ts index 5a2ab8ef55..cc0fa564ce 100644 --- a/packages/core/src/BinaryData/ObjectStore.manager.ts +++ b/packages/core/src/binary-data/object-store.manager.ts @@ -3,9 +3,9 @@ import fs from 'node:fs/promises'; import type { Readable } from 'node:stream'; import { v4 as uuid } from 'uuid'; +import { ObjectStoreService } from './object-store/object-store.service.ee'; import type { BinaryData } from './types'; import { binaryToBuffer } from './utils'; -import { ObjectStoreService } from '../ObjectStore/ObjectStore.service.ee'; @Service() export class ObjectStoreManager implements BinaryData.Manager { diff --git a/packages/core/test/ObjectStore.service.test.ts b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts similarity index 59% rename from packages/core/test/ObjectStore.service.test.ts rename to packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts index 9899ad17fc..ca8f4c0c94 100644 --- a/packages/core/test/ObjectStore.service.test.ts +++ b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts @@ -1,9 +1,10 @@ +import type { S3Config } from '@n8n/config'; import axios from 'axios'; import { mock } from 'jest-mock-extended'; import { Readable } from 'stream'; -import { ObjectStoreService } from '@/ObjectStore/ObjectStore.service.ee'; -import { writeBlockedMessage } from '@/ObjectStore/utils'; +import { ObjectStoreService } from '@/binary-data/object-store/object-store.service.ee'; +import { writeBlockedMessage } from '@/binary-data/object-store/utils'; jest.mock('axios'); @@ -18,6 +19,12 @@ const mockError = new Error('Something went wrong!'); const fileId = 'workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32'; const mockBuffer = Buffer.from('Test data'); +const s3Config = mock({ + host: mockHost, + bucket: mockBucket, + credentials: mockCredentials, + protocol: 'https', +}); const toDeletionXml = (filename: string) => ` ${filename} @@ -25,10 +32,13 @@ const toDeletionXml = (filename: string) => ` let objectStoreService: ObjectStoreService; +const now = new Date('2024-02-01T01:23:45.678Z'); +jest.useFakeTimers({ now }); + beforeEach(async () => { - objectStoreService = new ObjectStoreService(mock()); + objectStoreService = new ObjectStoreService(mock(), s3Config); mockAxios.request.mockResolvedValueOnce({ status: 200 }); // for checkConnection - await objectStoreService.init(mockHost, mockBucket, mockCredentials); + await objectStoreService.init(); jest.restoreAllMocks(); }); @@ -40,17 +50,17 @@ describe('checkConnection()', () => { await objectStoreService.checkConnection(); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'HEAD', - url: `https://${mockHost}/${mockBucket.name}`, - headers: expect.objectContaining({ - 'X-Amz-Content-Sha256': expect.any(String), - 'X-Amz-Date': expect.any(String), - Authorization: expect.any(String), - }), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'HEAD', + url: 'https://s3.us-east-1.amazonaws.com/test-bucket', + headers: { + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a5240c11a706e9e6c60e7033a848fc934911b12330e5a4609b0b943f97d9781b', + }, + }); }); it('should throw an error on request failure', async () => { @@ -70,18 +80,17 @@ describe('getMetadata()', () => { await objectStoreService.getMetadata(fileId); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'HEAD', - url: `${mockUrl}/${fileId}`, - headers: expect.objectContaining({ - Host: mockHost, - 'X-Amz-Content-Sha256': expect.any(String), - 'X-Amz-Date': expect.any(String), - Authorization: expect.any(String), - }), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'HEAD', + url: `${mockUrl}/${fileId}`, + headers: { + Host: mockHost, + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=60e11c39580ad7dd3a3d549523e7115cdff018540f24c6412ed40053e52a21d0', + }, + }); }); it('should throw an error on request failure', async () => { @@ -101,19 +110,22 @@ describe('put()', () => { await objectStoreService.put(fileId, mockBuffer, metadata); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'PUT', - url: `${mockUrl}/${fileId}`, - headers: expect.objectContaining({ - 'Content-Length': mockBuffer.length, - 'Content-MD5': expect.any(String), - 'x-amz-meta-filename': metadata.fileName, - 'Content-Type': metadata.mimeType, - }), - data: mockBuffer, - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'PUT', + url: 'https://s3.us-east-1.amazonaws.com/test-bucket/workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32', + headers: { + 'Content-Length': 9, + 'Content-MD5': 'yh6gLBC3w39CW5t92G1eEQ==', + 'x-amz-meta-filename': 'file.txt', + 'Content-Type': 'text/plain', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e27c8214be8b7cf5bccc7c08247e3cb0c1514a48ee1f63197fe4ef3ef51d7e6f', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-meta-filename, Signature=6b0fbb51a35dbfa73ac79a964ffc7203b40517a062efc5b01f5f9b7ad553fa7a', + }, + data: mockBuffer, + }); }); it('should block if read-only', async () => { @@ -152,13 +164,18 @@ describe('get()', () => { const result = await objectStoreService.get(fileId, { mode: 'buffer' }); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: `${mockUrl}/${fileId}`, - responseType: 'arraybuffer', - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'GET', + url: `${mockUrl}/${fileId}`, + responseType: 'arraybuffer', + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5f69680786e0ad9f0a0324eb5e4b8fe8c78562afc924489ea423632a2ad2187d', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); expect(Buffer.isBuffer(result)).toBe(true); }); @@ -168,13 +185,18 @@ describe('get()', () => { const result = await objectStoreService.get(fileId, { mode: 'stream' }); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: `${mockUrl}/${fileId}`, - responseType: 'stream', - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'GET', + url: `${mockUrl}/${fileId}`, + responseType: 'stream', + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3ef579ebe2ae89303a89c0faf3ce8ef8e907295dc538d59e95bcf35481c0d03e', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); expect(result instanceof Readable).toBe(true); }); @@ -194,12 +216,17 @@ describe('deleteOne()', () => { await objectStoreService.deleteOne(fileId); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'DELETE', - url: `${mockUrl}/${fileId}`, - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'DELETE', + url: `${mockUrl}/${fileId}`, + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4ad61b1b4da335c6c49772d28e54a301f787d199c9403055b217f890f7aec7fc', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); }); it('should throw an error on request failure', async () => { @@ -232,19 +259,21 @@ describe('deleteMany()', () => { await objectStoreService.deleteMany(prefix); - expect(objectStoreService.list).toHaveBeenCalledWith(prefix); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'POST', - url: `${mockUrl}/?delete`, - headers: expect.objectContaining({ - 'Content-Type': 'application/xml', - 'Content-Length': expect.any(Number), - 'Content-MD5': expect.any(String), - }), - data: toDeletionXml(fileName), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'POST', + url: `${mockUrl}?delete=`, + headers: { + 'Content-Type': 'application/xml', + 'Content-Length': 55, + 'Content-MD5': 'ybYDrpQxwYvNIGBQs7PJNA==', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': '5708e5c935cb75eb528e41ef1548e08b26c5b3b7504b67dc911abc1ff1881f76', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date, Signature=039168f10927b31624f3a5edae8eb4c89405f7c594eb2d6e00257c1462363f99', + }, + data: toDeletionXml(fileName), + }); }); it('should not send a deletion request if no prefix match', async () => { diff --git a/packages/core/src/ObjectStore/ObjectStore.service.ee.ts b/packages/core/src/binary-data/object-store/object-store.service.ee.ts similarity index 74% rename from packages/core/src/ObjectStore/ObjectStore.service.ee.ts rename to packages/core/src/binary-data/object-store/object-store.service.ee.ts index e1b98c6f50..5561bd61db 100644 --- a/packages/core/src/ObjectStore/ObjectStore.service.ee.ts +++ b/packages/core/src/binary-data/object-store/object-store.service.ee.ts @@ -1,6 +1,7 @@ +import { S3Config } from '@n8n/config'; import { Service } from '@n8n/di'; import { sign } from 'aws4'; -import type { Request as Aws4Options, Credentials as Aws4Credentials } from 'aws4'; +import type { Request as Aws4Options } from 'aws4'; import axios from 'axios'; import type { AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig, Method } from 'axios'; import { ApplicationError } from 'n8n-workflow'; @@ -9,43 +10,41 @@ import type { Readable } from 'stream'; import { Logger } from '@/logging/logger'; -import type { - Bucket, - ConfigSchemaCredentials, - ListPage, - MetadataResponseHeaders, - RawListPage, - RequestOptions, -} from './types'; +import type { ListPage, MetadataResponseHeaders, RawListPage, RequestOptions } from './types'; import { isStream, parseXml, writeBlockedMessage } from './utils'; -import type { BinaryData } from '../BinaryData/types'; +import type { BinaryData } from '../types'; @Service() export class ObjectStoreService { - private host = ''; - - private bucket: Bucket = { region: '', name: '' }; - - private credentials: Aws4Credentials = { accessKeyId: '', secretAccessKey: '' }; + private baseUrl: URL; private isReady = false; private isReadOnly = false; - constructor(private readonly logger: Logger) {} + constructor( + private readonly logger: Logger, + private readonly s3Config: S3Config, + ) { + const { host, bucket, protocol } = s3Config; - async init(host: string, bucket: Bucket, credentials: ConfigSchemaCredentials) { - this.host = host; - this.bucket.name = bucket.name; - this.bucket.region = bucket.region; + if (host === '') { + throw new ApplicationError( + 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', + ); + } - this.credentials = { - accessKeyId: credentials.accessKey, - secretAccessKey: credentials.accessSecret, - }; + if (bucket.name === '') { + throw new ApplicationError( + 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', + ); + } + this.baseUrl = new URL(`${protocol}://${host}/${bucket.name}`); + } + + async init() { await this.checkConnection(); - this.setReady(true); } @@ -65,7 +64,7 @@ export class ObjectStoreService { async checkConnection() { if (this.isReady) return; - return await this.request('HEAD', this.host, this.bucket.name); + return await this.request('HEAD', ''); } /** @@ -84,9 +83,7 @@ export class ObjectStoreService { if (metadata.fileName) headers['x-amz-meta-filename'] = metadata.fileName; if (metadata.mimeType) headers['Content-Type'] = metadata.mimeType; - const path = `/${this.bucket.name}/${filename}`; - - return await this.request('PUT', this.host, path, { headers, body: buffer }); + return await this.request('PUT', filename, { headers, body: buffer }); } /** @@ -97,9 +94,7 @@ export class ObjectStoreService { async get(fileId: string, { mode }: { mode: 'buffer' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' | 'buffer' }) { - const path = `${this.bucket.name}/${fileId}`; - - const { data } = await this.request('GET', this.host, path, { + const { data } = await this.request('GET', fileId, { responseType: mode === 'buffer' ? 'arraybuffer' : 'stream', }); @@ -116,9 +111,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html */ async getMetadata(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - const response = await this.request('HEAD', this.host, path); + const response = await this.request('HEAD', fileId); return response.headers as MetadataResponseHeaders; } @@ -129,9 +122,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */ async deleteOne(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - return await this.request('DELETE', this.host, path); + return await this.request('DELETE', fileId); } /** @@ -154,9 +145,7 @@ export class ObjectStoreService { 'Content-MD5': createHash('md5').update(body).digest('base64'), }; - const path = `${this.bucket.name}/?delete`; - - return await this.request('POST', this.host, path, { headers, body }); + return await this.request('POST', '', { headers, body, qs: { delete: '' } }); } /** @@ -192,7 +181,7 @@ export class ObjectStoreService { if (nextPageToken) qs['continuation-token'] = nextPageToken; - const { data } = await this.request('GET', this.host, this.bucket.name, { qs }); + const { data } = await this.request('GET', '', { qs }); if (typeof data !== 'string') { throw new TypeError(`Expected XML string but received ${typeof data}`); @@ -215,18 +204,6 @@ export class ObjectStoreService { return page as ListPage; } - private toPath(rawPath: string, qs?: Record) { - const path = rawPath.startsWith('/') ? rawPath : `/${rawPath}`; - - if (!qs) return path; - - const qsParams = Object.entries(qs) - .map(([key, value]) => `${key}=${value}`) - .join('&'); - - return path.concat(`?${qsParams}`); - } - private async blockWrite(filename: string): Promise { const logMessage = writeBlockedMessage(filename); @@ -243,28 +220,37 @@ export class ObjectStoreService { private async request( method: Method, - host: string, rawPath = '', { qs, headers, body, responseType }: RequestOptions = {}, ) { - const path = this.toPath(rawPath, qs); + const url = new URL(this.baseUrl); + if (rawPath && rawPath !== '/') { + url.pathname = `${url.pathname}/${rawPath}`; + } + Object.entries(qs ?? {}).forEach(([key, value]) => { + url.searchParams.set(key, String(value)); + }); const optionsToSign: Aws4Options = { method, service: 's3', - region: this.bucket.region, - host, - path, + region: this.s3Config.bucket.region, + host: this.s3Config.host, + path: `${url.pathname}${url.search}`, }; if (headers) optionsToSign.headers = headers; if (body) optionsToSign.body = body; - const signedOptions = sign(optionsToSign, this.credentials); + const { accessKey, accessSecret } = this.s3Config.credentials; + const signedOptions = sign(optionsToSign, { + accessKeyId: accessKey, + secretAccessKey: accessSecret, + }); const config: AxiosRequestConfig = { method, - url: `https://${host}${path}`, + url: url.toString(), headers: signedOptions.headers, }; diff --git a/packages/core/src/ObjectStore/types.ts b/packages/core/src/binary-data/object-store/types.ts similarity index 81% rename from packages/core/src/ObjectStore/types.ts rename to packages/core/src/binary-data/object-store/types.ts index d0b7ab0713..20390cf243 100644 --- a/packages/core/src/ObjectStore/types.ts +++ b/packages/core/src/binary-data/object-store/types.ts @@ -1,6 +1,6 @@ import type { AxiosResponseHeaders, ResponseType } from 'axios'; -import type { BinaryData } from '../BinaryData/types'; +import type { BinaryData } from '../types'; export type RawListPage = { listBucketResult: { @@ -24,8 +24,6 @@ type Item = { export type ListPage = Omit & { contents: Item[] }; -export type Bucket = { region: string; name: string }; - export type RequestOptions = { qs?: Record; headers?: Record; @@ -38,5 +36,3 @@ export type MetadataResponseHeaders = AxiosResponseHeaders & { 'content-type'?: string; 'x-amz-meta-filename'?: string; } & BinaryData.PreWriteMetadata; - -export type ConfigSchemaCredentials = { accessKey: string; accessSecret: string }; diff --git a/packages/core/src/ObjectStore/utils.ts b/packages/core/src/binary-data/object-store/utils.ts similarity index 100% rename from packages/core/src/ObjectStore/utils.ts rename to packages/core/src/binary-data/object-store/utils.ts diff --git a/packages/core/src/BinaryData/types.ts b/packages/core/src/binary-data/types.ts similarity index 100% rename from packages/core/src/BinaryData/types.ts rename to packages/core/src/binary-data/types.ts diff --git a/packages/core/src/BinaryData/utils.ts b/packages/core/src/binary-data/utils.ts similarity index 100% rename from packages/core/src/BinaryData/utils.ts rename to packages/core/src/binary-data/utils.ts diff --git a/packages/core/src/Constants.ts b/packages/core/src/constants.ts similarity index 51% rename from packages/core/src/Constants.ts rename to packages/core/src/constants.ts index 82a39b07cd..bd44e25d10 100644 --- a/packages/core/src/Constants.ts +++ b/packages/core/src/constants.ts @@ -1,6 +1,3 @@ -import type { INodeProperties } from 'n8n-workflow'; -import { cronNodeOptions } from 'n8n-workflow'; - const { NODE_ENV } = process.env; export const inProduction = NODE_ENV === 'production'; export const inDevelopment = !NODE_ENV || NODE_ENV === 'development'; @@ -11,8 +8,6 @@ export const PLACEHOLDER_EMPTY_WORKFLOW_ID = '__EMPTY__'; export const HTTP_REQUEST_NODE_TYPE = 'n8n-nodes-base.httpRequest'; export const HTTP_REQUEST_TOOL_NODE_TYPE = '@n8n/n8n-nodes-langchain.toolHttpRequest'; -export const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; - export const RESTRICT_FILE_ACCESS_TO = 'N8N_RESTRICT_FILE_ACCESS_TO'; export const BLOCK_FILE_ACCESS_TO_N8N_FILES = 'N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES'; export const CONFIG_FILES = 'N8N_CONFIG_FILES'; @@ -20,29 +15,10 @@ export const BINARY_DATA_STORAGE_PATH = 'N8N_BINARY_DATA_STORAGE_PATH'; export const UM_EMAIL_TEMPLATES_INVITE = 'N8N_UM_EMAIL_TEMPLATES_INVITE'; export const UM_EMAIL_TEMPLATES_PWRESET = 'N8N_UM_EMAIL_TEMPLATES_PWRESET'; -export const commonPollingParameters: INodeProperties[] = [ - { - displayName: 'Poll Times', - name: 'pollTimes', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - multipleValueButtonText: 'Add Poll Time', - }, - default: { item: [{ mode: 'everyMinute' }] }, - description: 'Time at which polling should occur', - placeholder: 'Add Poll Time', - options: cronNodeOptions, - }, -]; - -export const commonCORSParameters: INodeProperties[] = [ - { - displayName: 'Allowed Origins (CORS)', - name: 'allowedOrigins', - type: 'string', - default: '*', - description: - 'Comma-separated list of URLs allowed for cross-origin non-preflight requests. Use * (default) to allow all origins.', - }, -]; +export const CREDENTIAL_ERRORS = { + NO_DATA: 'No data is set on this credentials.', + DECRYPTION_FAILED: + 'Credentials could not be decrypted. The likely reason is that a different "encryptionKey" was used to encrypt the data.', + INVALID_JSON: 'Decrypted credentials data is not valid JSON.', + INVALID_DATA: 'Credentials data is not in a valid format.', +}; diff --git a/packages/core/src/Credentials.ts b/packages/core/src/credentials.ts similarity index 54% rename from packages/core/src/Credentials.ts rename to packages/core/src/credentials.ts index f5f8eb834f..f2d10df156 100644 --- a/packages/core/src/Credentials.ts +++ b/packages/core/src/credentials.ts @@ -1,8 +1,20 @@ import { Container } from '@n8n/di'; import type { ICredentialDataDecryptedObject, ICredentialsEncrypted } from 'n8n-workflow'; import { ApplicationError, ICredentials, jsonParse } from 'n8n-workflow'; +import * as a from 'node:assert'; -import { Cipher } from './Cipher'; +import { CREDENTIAL_ERRORS } from '@/constants'; +import { Cipher } from '@/encryption/cipher'; +import { isObjectLiteral } from '@/utils'; + +export class CredentialDataError extends ApplicationError { + constructor({ name, type, id }: Credentials, message: string, cause?: unknown) { + super(message, { + extra: { name, type, id }, + cause, + }); + } +} export class Credentials< T extends object = ICredentialDataDecryptedObject, @@ -13,6 +25,8 @@ export class Credentials< * Sets new credential object */ setData(data: T): void { + a.ok(isObjectLiteral(data)); + this.data = this.cipher.encrypt(data); } @@ -21,17 +35,20 @@ export class Credentials< */ getData(): T { if (this.data === undefined) { - throw new ApplicationError('No data is set so nothing can be returned.'); + throw new CredentialDataError(this, CREDENTIAL_ERRORS.NO_DATA); + } + + let decryptedData: string; + try { + decryptedData = this.cipher.decrypt(this.data); + } catch (cause) { + throw new CredentialDataError(this, CREDENTIAL_ERRORS.DECRYPTION_FAILED, cause); } try { - const decryptedData = this.cipher.decrypt(this.data); - return jsonParse(decryptedData); - } catch (e) { - throw new ApplicationError( - 'Credentials could not be decrypted. The likely reason is that a different "encryptionKey" was used to encrypt the data.', - ); + } catch (cause) { + throw new CredentialDataError(this, CREDENTIAL_ERRORS.INVALID_JSON, cause); } } diff --git a/packages/core/test/Cipher.test.ts b/packages/core/src/encryption/__tests__/cipher.test.ts similarity index 87% rename from packages/core/test/Cipher.test.ts rename to packages/core/src/encryption/__tests__/cipher.test.ts index 7f6bcdedf3..c1e14a9be0 100644 --- a/packages/core/test/Cipher.test.ts +++ b/packages/core/src/encryption/__tests__/cipher.test.ts @@ -1,9 +1,9 @@ import { Container } from '@n8n/di'; -import { Cipher } from '@/Cipher'; -import { InstanceSettings } from '@/InstanceSettings'; +import { InstanceSettings } from '@/instance-settings'; +import { mockInstance } from '@test/utils'; -import { mockInstance } from './utils'; +import { Cipher } from '../cipher'; describe('Cipher', () => { mockInstance(InstanceSettings, { encryptionKey: 'test_key' }); diff --git a/packages/core/src/Cipher.ts b/packages/core/src/encryption/cipher.ts similarity index 96% rename from packages/core/src/Cipher.ts rename to packages/core/src/encryption/cipher.ts index 812c8f452a..248ca0317b 100644 --- a/packages/core/src/Cipher.ts +++ b/packages/core/src/encryption/cipher.ts @@ -1,7 +1,7 @@ import { Service } from '@n8n/di'; import { createHash, createCipheriv, createDecipheriv, randomBytes } from 'crypto'; -import { InstanceSettings } from './InstanceSettings'; +import { InstanceSettings } from '@/instance-settings'; // Data encrypted by CryptoJS always starts with these bytes const RANDOM_BYTES = Buffer.from('53616c7465645f5f', 'hex'); diff --git a/packages/core/src/encryption/index.ts b/packages/core/src/encryption/index.ts new file mode 100644 index 0000000000..b2ab9e782e --- /dev/null +++ b/packages/core/src/encryption/index.ts @@ -0,0 +1 @@ +export { Cipher } from './cipher'; diff --git a/packages/core/test/error-reporter.test.ts b/packages/core/src/errors/__tests__/error-reporter.test.ts similarity index 76% rename from packages/core/test/error-reporter.test.ts rename to packages/core/src/errors/__tests__/error-reporter.test.ts index 9edc27f15c..ff260ea1f7 100644 --- a/packages/core/test/error-reporter.test.ts +++ b/packages/core/src/errors/__tests__/error-reporter.test.ts @@ -4,9 +4,10 @@ import { AxiosError } from 'axios'; import { mock } from 'jest-mock-extended'; import { ApplicationError } from 'n8n-workflow'; -import { ErrorReporter } from '@/error-reporter'; import type { Logger } from '@/logging/logger'; +import { ErrorReporter } from '../error-reporter'; + jest.mock('@sentry/node', () => ({ init: jest.fn(), setTag: jest.fn(), @@ -101,6 +102,37 @@ describe('ErrorReporter', () => { const result = await errorReporter.beforeSend(event, { originalException }); expect(result).toBeNull(); }); + + describe('beforeSendFilter', () => { + const newErrorReportedWithBeforeSendFilter = (beforeSendFilter: jest.Mock) => { + const errorReporter = new ErrorReporter(mock()); + // @ts-expect-error - beforeSendFilter is private + errorReporter.beforeSendFilter = beforeSendFilter; + return errorReporter; + }; + + it('should filter out based on the beforeSendFilter', async () => { + const beforeSendFilter = jest.fn().mockReturnValue(true); + const errorReporter = newErrorReportedWithBeforeSendFilter(beforeSendFilter); + const hint = { originalException: new Error() }; + + const result = await errorReporter.beforeSend(event, hint); + + expect(result).toBeNull(); + expect(beforeSendFilter).toHaveBeenCalledWith(event, hint); + }); + + it('should not filter out when beforeSendFilter returns false', async () => { + const beforeSendFilter = jest.fn().mockReturnValue(false); + const errorReporter = newErrorReportedWithBeforeSendFilter(beforeSendFilter); + const hint = { originalException: new Error() }; + + const result = await errorReporter.beforeSend(event, hint); + + expect(result).toEqual(event); + expect(beforeSendFilter).toHaveBeenCalledWith(event, hint); + }); + }); }); describe('error', () => { diff --git a/packages/core/src/error-reporter.ts b/packages/core/src/errors/error-reporter.ts similarity index 66% rename from packages/core/src/error-reporter.ts rename to packages/core/src/errors/error-reporter.ts index 0bc0f6058e..3c0bffcd4b 100644 --- a/packages/core/src/error-reporter.ts +++ b/packages/core/src/errors/error-reporter.ts @@ -6,8 +6,21 @@ import { AxiosError } from 'axios'; import { ApplicationError, ExecutionCancelledError, type ReportingOptions } from 'n8n-workflow'; import { createHash } from 'node:crypto'; -import type { InstanceType } from './InstanceSettings'; -import { Logger } from './logging/logger'; +import type { InstanceType } from '@/instance-settings'; +import { Logger } from '@/logging/logger'; + +type ErrorReporterInitOptions = { + serverType: InstanceType | 'task_runner'; + dsn: string; + release: string; + environment: string; + serverName: string; + /** + * Function to allow filtering out errors before they are sent to Sentry. + * Return true if the error should be filtered out. + */ + beforeSendFilter?: (event: ErrorEvent, hint: EventHint) => boolean; +}; @Service() export class ErrorReporter { @@ -16,6 +29,8 @@ export class ErrorReporter { private report: (error: Error | string, options?: ReportingOptions) => void; + private beforeSendFilter?: (event: ErrorEvent, hint: EventHint) => boolean; + constructor(private readonly logger: Logger) { // eslint-disable-next-line @typescript-eslint/unbound-method this.report = this.defaultReport; @@ -44,7 +59,14 @@ export class ErrorReporter { await close(timeoutInMs); } - async init(instanceType: InstanceType | 'task_runner', dsn: string) { + async init({ + beforeSendFilter, + dsn, + serverType, + release, + environment, + serverName, + }: ErrorReporterInitOptions) { process.on('uncaughtException', (error) => { this.error(error); }); @@ -54,12 +76,6 @@ export class ErrorReporter { // Collect longer stacktraces Error.stackTraceLimit = 50; - const { - N8N_VERSION: release, - ENVIRONMENT: environment, - DEPLOYMENT_NAME: serverName, - } = process.env; - const { init, captureException, setTag } = await import('@sentry/node'); const { requestDataIntegration, rewriteFramesIntegration } = await import('@sentry/node'); @@ -95,34 +111,37 @@ export class ErrorReporter { ], }); - setTag('server_type', instanceType); + setTag('server_type', serverType); this.report = (error, options) => captureException(error, options); + this.beforeSendFilter = beforeSendFilter; } - async beforeSend(event: ErrorEvent, { originalException }: EventHint) { + async beforeSend(event: ErrorEvent, hint: EventHint) { + let { originalException } = hint; + if (!originalException) return null; if (originalException instanceof Promise) { originalException = await originalException.catch((error) => error as Error); } - if (originalException instanceof AxiosError) return null; - if ( - originalException instanceof Error && - originalException.name === 'QueryFailedError' && - ['SQLITE_FULL', 'SQLITE_IOERR'].some((errMsg) => originalException.message.includes(errMsg)) + this.beforeSendFilter?.(event, { + ...hint, + originalException, + }) ) { return null; } - if (originalException instanceof ApplicationError) { - const { level, extra, tags } = originalException; - if (level === 'warning') return null; - event.level = level; - if (extra) event.extra = { ...event.extra, ...extra }; - if (tags) event.tags = { ...event.tags, ...tags }; + if (originalException instanceof AxiosError) return null; + + if (this.isIgnoredSqliteError(originalException)) return null; + if (this.isApplicationError(originalException)) { + if (this.isIgnoredApplicationError(originalException)) return null; + + this.extractEventDetailsFromApplicationError(event, originalException); } if ( @@ -164,4 +183,31 @@ export class ErrorReporter { if (typeof e === 'string') return new ApplicationError(e); return; } + + /** @returns true if the error should be filtered out */ + private isIgnoredSqliteError(error: unknown) { + return ( + error instanceof Error && + error.name === 'QueryFailedError' && + ['SQLITE_FULL', 'SQLITE_IOERR'].some((errMsg) => error.message.includes(errMsg)) + ); + } + + private isApplicationError(error: unknown): error is ApplicationError { + return error instanceof ApplicationError; + } + + private isIgnoredApplicationError(error: ApplicationError) { + return error.level === 'warning'; + } + + private extractEventDetailsFromApplicationError( + event: ErrorEvent, + originalException: ApplicationError, + ) { + const { level, extra, tags } = originalException; + event.level = level; + if (extra) event.extra = { ...event.extra, ...extra }; + if (tags) event.tags = { ...event.tags, ...tags }; + } } diff --git a/packages/core/src/errors/index.ts b/packages/core/src/errors/index.ts index 38cd481c25..b6571dd336 100644 --- a/packages/core/src/errors/index.ts +++ b/packages/core/src/errors/index.ts @@ -5,3 +5,5 @@ export { InvalidManagerError } from './invalid-manager.error'; export { InvalidExecutionMetadataError } from './invalid-execution-metadata.error'; export { UnrecognizedCredentialTypeError } from './unrecognized-credential-type.error'; export { UnrecognizedNodeTypeError } from './unrecognized-node-type.error'; + +export { ErrorReporter } from './error-reporter'; diff --git a/packages/core/src/errors/invalid-mode.error.ts b/packages/core/src/errors/invalid-mode.error.ts index 179582911a..1e3a4f3594 100644 --- a/packages/core/src/errors/invalid-mode.error.ts +++ b/packages/core/src/errors/invalid-mode.error.ts @@ -1,6 +1,6 @@ import { ApplicationError } from 'n8n-workflow'; -import { CONFIG_MODES } from '../BinaryData/utils'; +import { CONFIG_MODES } from '../binary-data/utils'; export class InvalidModeError extends ApplicationError { constructor() { diff --git a/packages/core/src/errors/workflow-has-issues.error.ts b/packages/core/src/errors/workflow-has-issues.error.ts new file mode 100644 index 0000000000..77f73131c2 --- /dev/null +++ b/packages/core/src/errors/workflow-has-issues.error.ts @@ -0,0 +1,7 @@ +import { WorkflowOperationError } from 'n8n-workflow'; + +export class WorkflowHasIssuesError extends WorkflowOperationError { + constructor() { + super('The workflow has issues and cannot be executed for that reason. Please fix them first.'); + } +} diff --git a/packages/core/src/__tests__/ActiveWorkflows.test.ts b/packages/core/src/execution-engine/__tests__/active-workflows.test.ts similarity index 96% rename from packages/core/src/__tests__/ActiveWorkflows.test.ts rename to packages/core/src/execution-engine/__tests__/active-workflows.test.ts index 410b4779ba..022ef022cf 100644 --- a/packages/core/src/__tests__/ActiveWorkflows.test.ts +++ b/packages/core/src/execution-engine/__tests__/active-workflows.test.ts @@ -1,6 +1,5 @@ import { mock } from 'jest-mock-extended'; import type { - IGetExecuteTriggerFunctions, INode, ITriggerResponse, IWorkflowExecuteAdditionalData, @@ -12,11 +11,13 @@ import type { } from 'n8n-workflow'; import { LoggerProxy, TriggerCloseError, WorkflowActivationError } from 'n8n-workflow'; -import { ActiveWorkflows } from '@/ActiveWorkflows'; -import type { ErrorReporter } from '@/error-reporter'; -import type { PollContext } from '@/node-execution-context'; -import type { ScheduledTaskManager } from '@/ScheduledTaskManager'; -import type { TriggersAndPollers } from '@/TriggersAndPollers'; +import type { ErrorReporter } from '@/errors/error-reporter'; + +import { ActiveWorkflows } from '../active-workflows'; +import type { IGetExecuteTriggerFunctions } from '../interfaces'; +import type { PollContext } from '../node-execution-context'; +import type { ScheduledTaskManager } from '../scheduled-task-manager'; +import type { TriggersAndPollers } from '../triggers-and-pollers'; describe('ActiveWorkflows', () => { const workflowId = 'test-workflow-id'; diff --git a/packages/core/src/execution-engine/__tests__/execution-lifecycle-hooks.test.ts b/packages/core/src/execution-engine/__tests__/execution-lifecycle-hooks.test.ts new file mode 100644 index 0000000000..fb85216b25 --- /dev/null +++ b/packages/core/src/execution-engine/__tests__/execution-lifecycle-hooks.test.ts @@ -0,0 +1,113 @@ +import { mock } from 'jest-mock-extended'; +import type { + IDataObject, + IExecuteResponsePromiseData, + INode, + IRun, + IRunExecutionData, + ITaskData, + IWorkflowBase, + Workflow, +} from 'n8n-workflow'; + +import type { + ExecutionLifecycleHookName, + ExecutionLifecyleHookHandlers, +} from '../execution-lifecycle-hooks'; +import { ExecutionLifecycleHooks } from '../execution-lifecycle-hooks'; + +describe('ExecutionLifecycleHooks', () => { + const executionId = '123'; + const workflowData = mock(); + + let hooks: ExecutionLifecycleHooks; + beforeEach(() => { + jest.clearAllMocks(); + hooks = new ExecutionLifecycleHooks('internal', executionId, workflowData); + }); + + describe('constructor()', () => { + it('should initialize with correct properties', () => { + expect(hooks.mode).toBe('internal'); + expect(hooks.executionId).toBe(executionId); + expect(hooks.workflowData).toBe(workflowData); + expect(hooks.handlers).toEqual({ + nodeExecuteAfter: [], + nodeExecuteBefore: [], + nodeFetchedData: [], + sendResponse: [], + workflowExecuteAfter: [], + workflowExecuteBefore: [], + }); + }); + }); + + describe('addHandler()', () => { + const hooksHandlers = + mock<{ + [K in keyof ExecutionLifecyleHookHandlers]: ExecutionLifecyleHookHandlers[K][number]; + }>(); + + const testCases: Array<{ + hook: ExecutionLifecycleHookName; + args: Parameters; + }> = [ + { hook: 'nodeExecuteBefore', args: ['testNode'] }, + { + hook: 'nodeExecuteAfter', + args: ['testNode', mock(), mock()], + }, + { hook: 'workflowExecuteBefore', args: [mock(), mock()] }, + { hook: 'workflowExecuteAfter', args: [mock(), mock()] }, + { hook: 'sendResponse', args: [mock()] }, + { hook: 'nodeFetchedData', args: ['workflow123', mock()] }, + ]; + + test.each(testCases)( + 'should add handlers to $hook hook and call them', + async ({ hook, args }) => { + hooks.addHandler(hook, hooksHandlers[hook]); + await hooks.runHook(hook, args); + expect(hooksHandlers[hook]).toHaveBeenCalledWith(...args); + }, + ); + }); + + describe('runHook()', () => { + it('should execute multiple hooks in order', async () => { + const executionOrder: string[] = []; + const hook1 = jest.fn().mockImplementation(async () => { + executionOrder.push('hook1'); + }); + const hook2 = jest.fn().mockImplementation(async () => { + executionOrder.push('hook2'); + }); + + hooks.addHandler('nodeExecuteBefore', hook1, hook2); + await hooks.runHook('nodeExecuteBefore', ['testNode']); + + expect(executionOrder).toEqual(['hook1', 'hook2']); + expect(hook1).toHaveBeenCalled(); + expect(hook2).toHaveBeenCalled(); + }); + + it('should maintain correct "this" context', async () => { + const hook = jest.fn().mockImplementation(async function (this: ExecutionLifecycleHooks) { + expect(this.executionId).toBe(executionId); + expect(this.mode).toBe('internal'); + }); + + hooks.addHandler('nodeExecuteBefore', hook); + await hooks.runHook('nodeExecuteBefore', ['testNode']); + + expect(hook).toHaveBeenCalled(); + }); + + it('should handle errors in hooks', async () => { + const errorHook = jest.fn().mockRejectedValue(new Error('Hook failed')); + hooks.addHandler('nodeExecuteBefore', errorHook); + + await expect(hooks.runHook('nodeExecuteBefore', ['testNode'])).rejects.toThrow('Hook failed'); + }); + }); +}); diff --git a/packages/core/test/RoutingNode.test.ts b/packages/core/src/execution-engine/__tests__/routing-node.test.ts similarity index 98% rename from packages/core/test/RoutingNode.test.ts rename to packages/core/src/execution-engine/__tests__/routing-node.test.ts index 45ef937803..a35ad24a8a 100644 --- a/packages/core/test/RoutingNode.test.ts +++ b/packages/core/src/execution-engine/__tests__/routing-node.test.ts @@ -21,10 +21,10 @@ import type { } from 'n8n-workflow'; import { NodeHelpers, Workflow } from 'n8n-workflow'; -import * as executionContexts from '@/node-execution-context'; -import { RoutingNode } from '@/RoutingNode'; +import * as executionContexts from '@/execution-engine/node-execution-context'; +import { NodeTypes } from '@test/helpers'; -import { NodeTypes } from './helpers'; +import { RoutingNode } from '../routing-node'; const postReceiveFunction1 = async function ( this: IExecuteSingleFunctions, @@ -744,14 +744,17 @@ describe('RoutingNode', () => { nodeTypes, }); - const routingNode = new RoutingNode( + const executeFunctions = mock(); + Object.assign(executeFunctions, { + runIndex, + additionalData, workflow, node, - connectionInputData, - runExecutionData ?? null, - additionalData, mode, - ); + connectionInputData, + runExecutionData, + }); + const routingNode = new RoutingNode(executeFunctions, nodeType); const executeSingleFunctions = getExecuteSingleFunctions( workflow, @@ -1947,15 +1950,6 @@ describe('RoutingNode', () => { nodeTypes, }); - const routingNode = new RoutingNode( - workflow, - node, - connectionInputData, - runExecutionData ?? null, - additionalData, - mode, - ); - const executeData = { data: {}, node, @@ -1963,6 +1957,18 @@ describe('RoutingNode', () => { } as IExecuteData; const executeFunctions = mock(); + Object.assign(executeFunctions, { + executeData, + inputData, + runIndex, + additionalData, + workflow, + node, + mode, + connectionInputData, + runExecutionData, + }); + const executeSingleFunctions = getExecuteSingleFunctions( workflow, runExecutionData, @@ -1971,7 +1977,6 @@ describe('RoutingNode', () => { itemIndex, ); - jest.spyOn(executionContexts, 'ExecuteContext').mockReturnValue(executeFunctions); jest .spyOn(executionContexts, 'ExecuteSingleContext') .mockReturnValue(executeSingleFunctions); @@ -2004,7 +2009,8 @@ describe('RoutingNode', () => { ? testData.input.node.parameters[parameterName] : (getNodeParameter(parameterName) ?? {}); - const result = await routingNode.runNode(inputData, runIndex, nodeType, executeData); + const routingNode = new RoutingNode(executeFunctions, nodeType); + const result = await routingNode.runNode(); if (testData.input.specialTestOptions?.sleepCalls) { expect(spy.mock.calls).toEqual(testData.input.specialTestOptions?.sleepCalls); diff --git a/packages/core/test/ScheduledTaskManager.test.ts b/packages/core/src/execution-engine/__tests__/scheduled-task-manager.test.ts similarity index 94% rename from packages/core/test/ScheduledTaskManager.test.ts rename to packages/core/src/execution-engine/__tests__/scheduled-task-manager.test.ts index 5166240856..4c39d3afbc 100644 --- a/packages/core/test/ScheduledTaskManager.test.ts +++ b/packages/core/src/execution-engine/__tests__/scheduled-task-manager.test.ts @@ -1,8 +1,9 @@ import { mock } from 'jest-mock-extended'; import type { Workflow } from 'n8n-workflow'; -import type { InstanceSettings } from '@/InstanceSettings'; -import { ScheduledTaskManager } from '@/ScheduledTaskManager'; +import type { InstanceSettings } from '@/instance-settings'; + +import { ScheduledTaskManager } from '../scheduled-task-manager'; describe('ScheduledTaskManager', () => { const instanceSettings = mock({ isLeader: true }); diff --git a/packages/core/test/SSHClientsManager.test.ts b/packages/core/src/execution-engine/__tests__/ssh-clients-manager.test.ts similarity index 97% rename from packages/core/test/SSHClientsManager.test.ts rename to packages/core/src/execution-engine/__tests__/ssh-clients-manager.test.ts index 132a54baef..d58fe22802 100644 --- a/packages/core/test/SSHClientsManager.test.ts +++ b/packages/core/src/execution-engine/__tests__/ssh-clients-manager.test.ts @@ -1,7 +1,7 @@ import type { SSHCredentials } from 'n8n-workflow'; import { Client } from 'ssh2'; -import { SSHClientsManager } from '@/SSHClientsManager'; +import { SSHClientsManager } from '../ssh-clients-manager'; describe('SSHClientsManager', () => { const credentials: SSHCredentials = { diff --git a/packages/core/test/TriggersAndPollers.test.ts b/packages/core/src/execution-engine/__tests__/triggers-and-pollers.test.ts similarity index 89% rename from packages/core/test/TriggersAndPollers.test.ts rename to packages/core/src/execution-engine/__tests__/triggers-and-pollers.test.ts index 27cc8b47d9..25b1fddb26 100644 --- a/packages/core/test/TriggersAndPollers.test.ts +++ b/packages/core/src/execution-engine/__tests__/triggers-and-pollers.test.ts @@ -9,11 +9,11 @@ import type { INodeType, INodeTypes, ITriggerFunctions, - WorkflowHooks, IRun, } from 'n8n-workflow'; -import { TriggersAndPollers } from '@/TriggersAndPollers'; +import { ExecutionLifecycleHooks } from '../execution-lifecycle-hooks'; +import { TriggersAndPollers } from '../triggers-and-pollers'; describe('TriggersAndPollers', () => { const node = mock(); @@ -23,15 +23,8 @@ describe('TriggersAndPollers', () => { }); const nodeTypes = mock(); const workflow = mock({ nodeTypes }); - const hookFunctions = mock({ - sendResponse: [], - workflowExecuteAfter: [], - }); - const additionalData = mock({ - hooks: { - hookFunctions, - }, - }); + const hooks = new ExecutionLifecycleHooks('internal', '123', mock()); + const additionalData = mock({ hooks }); const triggersAndPollers = new TriggersAndPollers(); beforeEach(() => { @@ -98,8 +91,7 @@ describe('TriggersAndPollers', () => { getMockTriggerFunctions()?.emit?.(mockEmitData, responsePromise); - expect(hookFunctions.sendResponse?.length).toBe(1); - await hookFunctions.sendResponse![0]?.({ testResponse: true }); + await hooks.runHook('sendResponse', [{ testResponse: true }]); expect(responsePromise.resolve).toHaveBeenCalledWith({ testResponse: true }); }); @@ -111,10 +103,10 @@ describe('TriggersAndPollers', () => { await runTriggerHelper('manual'); getMockTriggerFunctions()?.emit?.(mockEmitData, responsePromise, donePromise); - await hookFunctions.sendResponse![0]?.({ testResponse: true }); + await hooks.runHook('sendResponse', [{ testResponse: true }]); expect(responsePromise.resolve).toHaveBeenCalledWith({ testResponse: true }); - await hookFunctions.workflowExecuteAfter?.[0]?.(mockRunData, {}); + await hooks.runHook('workflowExecuteAfter', [mockRunData, {}]); expect(donePromise.resolve).toHaveBeenCalledWith(mockRunData); }); }); diff --git a/packages/core/test/WorkflowExecute.test.ts b/packages/core/src/execution-engine/__tests__/workflow-execute.test.ts similarity index 97% rename from packages/core/test/WorkflowExecute.test.ts rename to packages/core/src/execution-engine/__tests__/workflow-execute.test.ts index 6ab3afdaeb..10af4a1e7b 100644 --- a/packages/core/test/WorkflowExecute.test.ts +++ b/packages/core/src/execution-engine/__tests__/workflow-execute.test.ts @@ -1,3 +1,7 @@ +// Disable task runners until we have fixed the "run test workflows" test +// to mock the Code Node execution +process.env.N8N_RUNNERS_ENABLED = 'false'; + // NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ // If you update the tests, please update the diagrams as well. // If you add a test, please create a new diagram. @@ -37,13 +41,14 @@ import { Workflow, } from 'n8n-workflow'; -import { DirectedGraph } from '@/PartialExecutionUtils'; -import * as partialExecutionUtils from '@/PartialExecutionUtils'; -import { createNodeData, toITaskData } from '@/PartialExecutionUtils/__tests__/helpers'; -import { WorkflowExecute } from '@/WorkflowExecute'; +import * as Helpers from '@test/helpers'; +import { legacyWorkflowExecuteTests, v1WorkflowExecuteTests } from '@test/helpers/constants'; -import * as Helpers from './helpers'; -import { legacyWorkflowExecuteTests, v1WorkflowExecuteTests } from './helpers/constants'; +import type { ExecutionLifecycleHooks } from '../execution-lifecycle-hooks'; +import { DirectedGraph } from '../partial-execution-utils'; +import * as partialExecutionUtils from '../partial-execution-utils'; +import { createNodeData, toITaskData } from '../partial-execution-utils/__tests__/helpers'; +import { WorkflowExecute } from '../workflow-execute'; const nodeTypes = Helpers.NodeTypes(); @@ -1207,6 +1212,7 @@ describe('WorkflowExecute', () => { let runExecutionData: IRunExecutionData; let workflowExecute: WorkflowExecute; + let additionalData: IWorkflowExecuteAdditionalData; beforeEach(() => { runExecutionData = { @@ -1220,9 +1226,12 @@ describe('WorkflowExecute', () => { waitingExecutionSource: null, }, }; - workflowExecute = new WorkflowExecute(mock(), 'manual', runExecutionData); + additionalData = mock(); + additionalData.hooks = mock(); - jest.spyOn(workflowExecute, 'executeHook').mockResolvedValue(undefined); + workflowExecute = new WorkflowExecute(additionalData, 'manual', runExecutionData); + + jest.spyOn(additionalData.hooks, 'runHook').mockResolvedValue(undefined); jest.spyOn(workflowExecute, 'moveNodeMetadata').mockImplementation(); }); @@ -1290,7 +1299,7 @@ describe('WorkflowExecute', () => { // Verify static data handling expect(result).toBeDefined(); expect(workflowExecute.moveNodeMetadata).toHaveBeenCalled(); - expect(workflowExecute.executeHook).toHaveBeenCalledWith('workflowExecuteAfter', [ + expect(additionalData.hooks?.runHook).toHaveBeenCalledWith('workflowExecuteAfter', [ result, workflow.staticData, ]); diff --git a/packages/core/test/workflows/error_outputs.json b/packages/core/src/execution-engine/__tests__/workflows/error_outputs.json similarity index 100% rename from packages/core/test/workflows/error_outputs.json rename to packages/core/src/execution-engine/__tests__/workflows/error_outputs.json diff --git a/packages/core/test/workflows/paired_items_fix.json b/packages/core/src/execution-engine/__tests__/workflows/paired_items_fix.json similarity index 100% rename from packages/core/test/workflows/paired_items_fix.json rename to packages/core/src/execution-engine/__tests__/workflows/paired_items_fix.json diff --git a/packages/core/src/ActiveWorkflows.ts b/packages/core/src/execution-engine/active-workflows.ts similarity index 94% rename from packages/core/src/ActiveWorkflows.ts rename to packages/core/src/execution-engine/active-workflows.ts index abd544956a..1c1b5e2d14 100644 --- a/packages/core/src/ActiveWorkflows.ts +++ b/packages/core/src/execution-engine/active-workflows.ts @@ -1,7 +1,5 @@ import { Service } from '@n8n/di'; import type { - IGetExecutePollFunctions, - IGetExecuteTriggerFunctions, INode, ITriggerResponse, IWorkflowExecuteAdditionalData, @@ -18,11 +16,13 @@ import { WorkflowDeactivationError, } from 'n8n-workflow'; -import { ErrorReporter } from './error-reporter'; -import type { IWorkflowData } from './Interfaces'; -import { Logger } from './logging/logger'; -import { ScheduledTaskManager } from './ScheduledTaskManager'; -import { TriggersAndPollers } from './TriggersAndPollers'; +import { ErrorReporter } from '@/errors/error-reporter'; +import type { IWorkflowData } from '@/interfaces'; +import { Logger } from '@/logging/logger'; + +import type { IGetExecutePollFunctions, IGetExecuteTriggerFunctions } from './interfaces'; +import { ScheduledTaskManager } from './scheduled-task-manager'; +import { TriggersAndPollers } from './triggers-and-pollers'; @Service() export class ActiveWorkflows { diff --git a/packages/core/src/execution-engine/execution-lifecycle-hooks.ts b/packages/core/src/execution-engine/execution-lifecycle-hooks.ts new file mode 100644 index 0000000000..047d5a2075 --- /dev/null +++ b/packages/core/src/execution-engine/execution-lifecycle-hooks.ts @@ -0,0 +1,119 @@ +import type { + IDataObject, + IExecuteResponsePromiseData, + INode, + IRun, + IRunExecutionData, + ITaskData, + IWorkflowBase, + Workflow, + WorkflowExecuteMode, +} from 'n8n-workflow'; + +export type ExecutionLifecyleHookHandlers = { + nodeExecuteBefore: Array< + (this: ExecutionLifecycleHooks, nodeName: string) => Promise | void + >; + + nodeExecuteAfter: Array< + ( + this: ExecutionLifecycleHooks, + nodeName: string, + data: ITaskData, + executionData: IRunExecutionData, + ) => Promise | void + >; + + workflowExecuteBefore: Array< + ( + this: ExecutionLifecycleHooks, + workflow: Workflow, + data?: IRunExecutionData, + ) => Promise | void + >; + + workflowExecuteAfter: Array< + (this: ExecutionLifecycleHooks, data: IRun, newStaticData: IDataObject) => Promise | void + >; + + /** Used by trigger and webhook nodes to respond back to the request */ + sendResponse: Array< + (this: ExecutionLifecycleHooks, response: IExecuteResponsePromiseData) => Promise | void + >; + + /** + * Executed after a node fetches data + * - For a webhook node, after the node had been run. + * - For a http-request node, or any other node that makes http requests that still use the deprecated request* methods, after every successful http request +s */ + nodeFetchedData: Array< + (this: ExecutionLifecycleHooks, workflowId: string, node: INode) => Promise | void + >; +}; + +export type ExecutionLifecycleHookName = keyof ExecutionLifecyleHookHandlers; + +/** + * Contains hooks that trigger at specific events in an execution's lifecycle. Every hook has an array of callbacks to run. + * + * Common use cases include: + * - Saving execution progress to database + * - Pushing execution status updates to the frontend + * - Recording workflow statistics + * - Running external hooks for execution events + * - Error and Cancellation handling and cleanup + * + * @example + * ```typescript + * const hooks = new ExecutionLifecycleHooks(mode, executionId, workflowData); + * hooks.add('workflowExecuteAfter, async function(fullRunData) { + * await saveToDatabase(executionId, fullRunData); + *}); + * ``` + */ +export class ExecutionLifecycleHooks { + readonly handlers: ExecutionLifecyleHookHandlers = { + nodeExecuteAfter: [], + nodeExecuteBefore: [], + nodeFetchedData: [], + sendResponse: [], + workflowExecuteAfter: [], + workflowExecuteBefore: [], + }; + + constructor( + readonly mode: WorkflowExecuteMode, + readonly executionId: string, + readonly workflowData: IWorkflowBase, + ) {} + + addHandler( + hookName: Hook, + ...handlers: Array + ): void { + // @ts-expect-error FIX THIS + this.handlers[hookName].push(...handlers); + } + + async runHook< + Hook extends keyof ExecutionLifecyleHookHandlers, + Params extends unknown[] = Parameters< + Exclude[number] + >, + >(hookName: Hook, parameters: Params) { + const hooks = this.handlers[hookName]; + for (const hookFunction of hooks) { + const typedHookFunction = hookFunction as unknown as ( + this: ExecutionLifecycleHooks, + ...args: Params + ) => Promise; + await typedHookFunction.apply(this, parameters); + } + } +} + +declare module 'n8n-workflow' { + interface IWorkflowExecuteAdditionalData { + hooks?: ExecutionLifecycleHooks; + } +} diff --git a/packages/core/src/execution-engine/index.ts b/packages/core/src/execution-engine/index.ts new file mode 100644 index 0000000000..bb8adfb34c --- /dev/null +++ b/packages/core/src/execution-engine/index.ts @@ -0,0 +1,8 @@ +export * from './active-workflows'; +export * from './interfaces'; +export * from './routing-node'; +export * from './node-execution-context'; +export * from './partial-execution-utils'; +export * from './node-execution-context/utils/execution-metadata'; +export * from './workflow-execute'; +export { ExecutionLifecycleHooks } from './execution-lifecycle-hooks'; diff --git a/packages/core/src/execution-engine/interfaces.ts b/packages/core/src/execution-engine/interfaces.ts new file mode 100644 index 0000000000..809c032b4f --- /dev/null +++ b/packages/core/src/execution-engine/interfaces.ts @@ -0,0 +1,29 @@ +import type { + INode, + IPollFunctions, + ITriggerFunctions, + IWorkflowExecuteAdditionalData, + Workflow, + WorkflowActivateMode, + WorkflowExecuteMode, +} from 'n8n-workflow'; + +export interface IGetExecutePollFunctions { + ( + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + mode: WorkflowExecuteMode, + activation: WorkflowActivateMode, + ): IPollFunctions; +} + +export interface IGetExecuteTriggerFunctions { + ( + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + mode: WorkflowExecuteMode, + activation: WorkflowActivateMode, + ): ITriggerFunctions; +} diff --git a/packages/core/src/node-execution-context/__tests__/execute-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/execute-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/execute-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/execute-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/execute-single-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/execute-single-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/execute-single-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/execute-single-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/hook-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/hook-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/hook-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/hook-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/load-options-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/load-options-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/load-options-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/load-options-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/node-execution-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts similarity index 99% rename from packages/core/src/node-execution-context/__tests__/node-execution-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts index a0a368df4c..6a10c2bba5 100644 --- a/packages/core/src/node-execution-context/__tests__/node-execution-context.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts @@ -12,7 +12,7 @@ import type { } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; -import { InstanceSettings } from '@/InstanceSettings'; +import { InstanceSettings } from '@/instance-settings'; import { NodeExecutionContext } from '../node-execution-context'; diff --git a/packages/core/src/node-execution-context/__tests__/poll-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/poll-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/poll-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/poll-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/shared-tests.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/shared-tests.ts similarity index 98% rename from packages/core/src/node-execution-context/__tests__/shared-tests.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/shared-tests.ts index 8b1d23930b..de5edaad96 100644 --- a/packages/core/src/node-execution-context/__tests__/shared-tests.ts +++ b/packages/core/src/execution-engine/node-execution-context/__tests__/shared-tests.ts @@ -17,7 +17,7 @@ import type { } from 'n8n-workflow'; import { ApplicationError, NodeHelpers, WAIT_INDEFINITELY } from 'n8n-workflow'; -import { BinaryDataService } from '@/BinaryData/BinaryData.service'; +import { BinaryDataService } from '@/binary-data/binary-data.service'; import type { BaseExecuteContext } from '../base-execute-context'; @@ -40,7 +40,6 @@ export const describeCommonTests = ( executeData: IExecuteData; }, ) => { - // @ts-expect-error `additionalData` is private const additionalData = context.additionalData as MockProxy; describe('getExecutionCancelSignal', () => { diff --git a/packages/core/src/node-execution-context/__tests__/supply-data-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/supply-data-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/supply-data-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/supply-data-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/trigger-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/trigger-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/trigger-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/trigger-context.test.ts diff --git a/packages/core/src/node-execution-context/__tests__/webhook-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/webhook-context.test.ts similarity index 100% rename from packages/core/src/node-execution-context/__tests__/webhook-context.test.ts rename to packages/core/src/execution-engine/node-execution-context/__tests__/webhook-context.test.ts diff --git a/packages/core/src/node-execution-context/base-execute-context.ts b/packages/core/src/execution-engine/node-execution-context/base-execute-context.ts similarity index 94% rename from packages/core/src/node-execution-context/base-execute-context.ts rename to packages/core/src/execution-engine/node-execution-context/base-execute-context.ts index 0416870344..f2d0aa8653 100644 --- a/packages/core/src/node-execution-context/base-execute-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/base-execute-context.ts @@ -29,7 +29,7 @@ import { WorkflowDataProxy, } from 'n8n-workflow'; -import { BinaryDataService } from '@/BinaryData/BinaryData.service'; +import { BinaryDataService } from '@/binary-data/binary-data.service'; import { NodeExecutionContext } from './node-execution-context'; @@ -41,12 +41,12 @@ export class BaseExecuteContext extends NodeExecutionContext { node: INode, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, - protected readonly runExecutionData: IRunExecutionData, + readonly runExecutionData: IRunExecutionData, runIndex: number, - protected readonly connectionInputData: INodeExecutionData[], - protected readonly inputData: ITaskDataConnections, - protected readonly executeData: IExecuteData, - protected readonly abortSignal?: AbortSignal, + readonly connectionInputData: INodeExecutionData[], + readonly inputData: ITaskDataConnections, + readonly executeData: IExecuteData, + readonly abortSignal?: AbortSignal, ) { super(workflow, node, additionalData, mode, runExecutionData, runIndex); } diff --git a/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts b/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts new file mode 100644 index 0000000000..9aa99683da --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts @@ -0,0 +1,26 @@ +import { Container } from '@n8n/di'; +import type { ICredentialTestFunctions } from 'n8n-workflow'; + +import { Memoized } from '@/decorators'; +import { Logger } from '@/logging'; +// eslint-disable-next-line import/no-cycle +import { getSSHTunnelFunctions, proxyRequestToAxios } from '@/node-execute-functions'; + +export class CredentialTestContext implements ICredentialTestFunctions { + readonly helpers: ICredentialTestFunctions['helpers']; + + constructor() { + this.helpers = { + ...getSSHTunnelFunctions(), + request: async (uriOrObject: string | object, options?: object) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return await proxyRequestToAxios(undefined, undefined, undefined, uriOrObject, options); + }, + }; + } + + @Memoized + get logger() { + return Container.get(Logger); + } +} diff --git a/packages/core/src/node-execution-context/execute-context.ts b/packages/core/src/execution-engine/node-execution-context/execute-context.ts similarity index 91% rename from packages/core/src/node-execution-context/execute-context.ts rename to packages/core/src/execution-engine/node-execution-context/execute-context.ts index 089c3f500a..bf2e89f8ec 100644 --- a/packages/core/src/node-execution-context/execute-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/execute-context.ts @@ -28,19 +28,21 @@ import { copyInputItems, normalizeItems, constructExecutionMetaData, + getRequestHelperFunctions, + getSSHTunnelFunctions, +} from '@/node-execute-functions'; + +import { BaseExecuteContext } from './base-execute-context'; +import { assertBinaryData, getBinaryDataBuffer, copyBinaryFile, - getRequestHelperFunctions, getBinaryHelperFunctions, - getSSHTunnelFunctions, - getFileSystemHelperFunctions, - getCheckProcessedHelperFunctions, detectBinaryEncoding, -} from '@/NodeExecuteFunctions'; - -import { BaseExecuteContext } from './base-execute-context'; -import { getInputConnectionData } from './utils/getInputConnectionData'; +} from './utils/binary-helper-functions'; +import { getDeduplicationHelperFunctions } from './utils/deduplication-helper-functions'; +import { getFileSystemHelperFunctions } from './utils/file-system-helper-functions'; +import { getInputConnectionData } from './utils/get-input-connection-data'; export class ExecuteContext extends BaseExecuteContext implements IExecuteFunctions { readonly helpers: IExecuteFunctions['helpers']; @@ -91,7 +93,7 @@ export class ExecuteContext extends BaseExecuteContext implements IExecuteFuncti ...getBinaryHelperFunctions(additionalData, workflow.id), ...getSSHTunnelFunctions(), ...getFileSystemHelperFunctions(node), - ...getCheckProcessedHelperFunctions(workflow, node), + ...getDeduplicationHelperFunctions(workflow, node), assertBinaryData: (itemIndex, propertyName) => assertBinaryData(inputData, node, itemIndex, propertyName, 0), @@ -192,7 +194,7 @@ export class ExecuteContext extends BaseExecuteContext implements IExecuteFuncti } async sendResponse(response: IExecuteResponsePromiseData): Promise { - await this.additionalData.hooks?.executeHookFunctions('sendResponse', [response]); + await this.additionalData.hooks?.runHook('sendResponse', [response]); } /** @deprecated use ISupplyDataFunctions.addInputData */ diff --git a/packages/core/src/node-execution-context/execute-single-context.ts b/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts similarity index 96% rename from packages/core/src/node-execution-context/execute-single-context.ts rename to packages/core/src/execution-engine/node-execution-context/execute-single-context.ts index af837a12c5..acf342e0ac 100644 --- a/packages/core/src/node-execution-context/execute-single-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts @@ -14,16 +14,15 @@ import type { import { ApplicationError, createDeferredPromise, NodeConnectionType } from 'n8n-workflow'; // eslint-disable-next-line import/no-cycle +import { getRequestHelperFunctions, returnJsonArray } from '@/node-execute-functions'; + +import { BaseExecuteContext } from './base-execute-context'; import { assertBinaryData, detectBinaryEncoding, getBinaryDataBuffer, getBinaryHelperFunctions, - getRequestHelperFunctions, - returnJsonArray, -} from '@/NodeExecuteFunctions'; - -import { BaseExecuteContext } from './base-execute-context'; +} from './utils/binary-helper-functions'; export class ExecuteSingleContext extends BaseExecuteContext implements IExecuteSingleFunctions { readonly helpers: IExecuteSingleFunctions['helpers']; diff --git a/packages/core/src/node-execution-context/hook-context.ts b/packages/core/src/execution-engine/node-execution-context/hook-context.ts similarity index 97% rename from packages/core/src/node-execution-context/hook-context.ts rename to packages/core/src/execution-engine/node-execution-context/hook-context.ts index 102be563a1..28dbac802c 100644 --- a/packages/core/src/node-execution-context/hook-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/hook-context.ts @@ -16,7 +16,7 @@ import { getNodeWebhookUrl, getRequestHelperFunctions, getWebhookDescription, -} from '@/NodeExecuteFunctions'; +} from '@/node-execute-functions'; import { NodeExecutionContext } from './node-execution-context'; diff --git a/packages/core/src/node-execution-context/index.ts b/packages/core/src/execution-engine/node-execution-context/index.ts similarity index 62% rename from packages/core/src/node-execution-context/index.ts rename to packages/core/src/execution-engine/node-execution-context/index.ts index c3bcebbd44..0c6843d525 100644 --- a/packages/core/src/node-execution-context/index.ts +++ b/packages/core/src/execution-engine/node-execution-context/index.ts @@ -1,4 +1,6 @@ // eslint-disable-next-line import/no-cycle +export { CredentialTestContext } from './credentials-test-context'; +// eslint-disable-next-line import/no-cycle export { ExecuteContext } from './execute-context'; export { ExecuteSingleContext } from './execute-single-context'; export { HookContext } from './hook-context'; @@ -10,4 +12,7 @@ export { SupplyDataContext } from './supply-data-context'; export { TriggerContext } from './trigger-context'; export { WebhookContext } from './webhook-context'; -export { getAdditionalKeys } from './utils/getAdditionalKeys'; +export { getAdditionalKeys } from './utils/get-additional-keys'; +export { parseIncomingMessage } from './utils/parse-incoming-message'; +export { parseRequestObject } from './utils/parse-request-object'; +export * from './utils/binary-helper-functions'; diff --git a/packages/core/src/node-execution-context/load-options-context.ts b/packages/core/src/execution-engine/node-execution-context/load-options-context.ts similarity index 95% rename from packages/core/src/node-execution-context/load-options-context.ts rename to packages/core/src/execution-engine/node-execution-context/load-options-context.ts index c961b56c06..30c31eacce 100644 --- a/packages/core/src/node-execution-context/load-options-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/load-options-context.ts @@ -9,11 +9,11 @@ import type { Workflow, } from 'n8n-workflow'; -import { extractValue } from '@/ExtractValue'; // eslint-disable-next-line import/no-cycle -import { getRequestHelperFunctions, getSSHTunnelFunctions } from '@/NodeExecuteFunctions'; +import { getRequestHelperFunctions, getSSHTunnelFunctions } from '@/node-execute-functions'; import { NodeExecutionContext } from './node-execution-context'; +import { extractValue } from './utils/extract-value'; export class LoadOptionsContext extends NodeExecutionContext implements ILoadOptionsFunctions { readonly helpers: ILoadOptionsFunctions['helpers']; diff --git a/packages/core/src/node-execution-context/local-load-options-context.ts b/packages/core/src/execution-engine/node-execution-context/local-load-options-context.ts similarity index 97% rename from packages/core/src/node-execution-context/local-load-options-context.ts rename to packages/core/src/execution-engine/node-execution-context/local-load-options-context.ts index 39456ff966..dd96ab5f74 100644 --- a/packages/core/src/node-execution-context/local-load-options-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/local-load-options-context.ts @@ -35,6 +35,8 @@ export class LocalLoadOptionsContext implements ILocalLoadOptionsFunctions { if (selectedWorkflowNode) { const selectedSingleNodeWorkflow = new Workflow({ + id: dbWorkflow.id, + name: dbWorkflow.name, nodes: [selectedWorkflowNode], connections: {}, active: false, diff --git a/packages/core/src/node-execution-context/node-execution-context.ts b/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts similarity index 92% rename from packages/core/src/node-execution-context/node-execution-context.ts rename to packages/core/src/execution-engine/node-execution-context/node-execution-context.ts index 45a205637c..662e24d6b6 100644 --- a/packages/core/src/node-execution-context/node-execution-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts @@ -28,29 +28,29 @@ import { NodeOperationError, } from 'n8n-workflow'; -import { HTTP_REQUEST_NODE_TYPE, HTTP_REQUEST_TOOL_NODE_TYPE } from '@/Constants'; +import { HTTP_REQUEST_NODE_TYPE, HTTP_REQUEST_TOOL_NODE_TYPE } from '@/constants'; import { Memoized } from '@/decorators'; -import { extractValue } from '@/ExtractValue'; -import { InstanceSettings } from '@/InstanceSettings'; +import { InstanceSettings } from '@/instance-settings'; import { Logger } from '@/logging/logger'; -import { cleanupParameterData } from './utils/cleanupParameterData'; -import { ensureType } from './utils/ensureType'; -import { getAdditionalKeys } from './utils/getAdditionalKeys'; -import { validateValueAgainstSchema } from './utils/validateValueAgainstSchema'; +import { cleanupParameterData } from './utils/cleanup-parameter-data'; +import { ensureType } from './utils/ensure-type'; +import { extractValue } from './utils/extract-value'; +import { getAdditionalKeys } from './utils/get-additional-keys'; +import { validateValueAgainstSchema } from './utils/validate-value-against-schema'; export abstract class NodeExecutionContext implements Omit { protected readonly instanceSettings = Container.get(InstanceSettings); constructor( - protected readonly workflow: Workflow, - protected readonly node: INode, - protected readonly additionalData: IWorkflowExecuteAdditionalData, - protected readonly mode: WorkflowExecuteMode, - protected readonly runExecutionData: IRunExecutionData | null = null, - protected readonly runIndex = 0, - protected readonly connectionInputData: INodeExecutionData[] = [], - protected readonly executeData?: IExecuteData, + readonly workflow: Workflow, + readonly node: INode, + readonly additionalData: IWorkflowExecuteAdditionalData, + readonly mode: WorkflowExecuteMode, + readonly runExecutionData: IRunExecutionData | null = null, + readonly runIndex = 0, + readonly connectionInputData: INodeExecutionData[] = [], + readonly executeData?: IExecuteData, ) {} @Memoized @@ -79,18 +79,24 @@ export abstract class NodeExecutionContext implements Omit { throw new ApplicationError('Overwrite PollContext.__emit function'); diff --git a/packages/core/src/node-execution-context/supply-data-context.ts b/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts similarity index 93% rename from packages/core/src/node-execution-context/supply-data-context.ts rename to packages/core/src/execution-engine/node-execution-context/supply-data-context.ts index be1f63b56c..33e2723a20 100644 --- a/packages/core/src/node-execution-context/supply-data-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts @@ -21,22 +21,24 @@ import { createDeferredPromise } from 'n8n-workflow'; // eslint-disable-next-line import/no-cycle import { - assertBinaryData, constructExecutionMetaData, copyInputItems, - detectBinaryEncoding, - getBinaryDataBuffer, - getBinaryHelperFunctions, - getCheckProcessedHelperFunctions, - getFileSystemHelperFunctions, getRequestHelperFunctions, getSSHTunnelFunctions, normalizeItems, returnJsonArray, -} from '@/NodeExecuteFunctions'; +} from '@/node-execute-functions'; import { BaseExecuteContext } from './base-execute-context'; -import { getInputConnectionData } from './utils/getInputConnectionData'; +import { + assertBinaryData, + detectBinaryEncoding, + getBinaryDataBuffer, + getBinaryHelperFunctions, +} from './utils/binary-helper-functions'; +import { getDeduplicationHelperFunctions } from './utils/deduplication-helper-functions'; +import { getFileSystemHelperFunctions } from './utils/file-system-helper-functions'; +import { getInputConnectionData } from './utils/get-input-connection-data'; export class SupplyDataContext extends BaseExecuteContext implements ISupplyDataFunctions { readonly helpers: ISupplyDataFunctions['helpers']; @@ -83,7 +85,7 @@ export class SupplyDataContext extends BaseExecuteContext implements ISupplyData ...getSSHTunnelFunctions(), ...getFileSystemHelperFunctions(node), ...getBinaryHelperFunctions(additionalData, workflow.id), - ...getCheckProcessedHelperFunctions(workflow, node), + ...getDeduplicationHelperFunctions(workflow, node), assertBinaryData: (itemIndex, propertyName) => assertBinaryData(inputData, node, itemIndex, propertyName, 0), getBinaryDataBuffer: async (itemIndex, propertyName) => @@ -256,12 +258,12 @@ export class SupplyDataContext extends BaseExecuteContext implements ISupplyData } runExecutionData.resultData.runData[nodeName][currentNodeRunIndex] = taskData; - await additionalData.hooks?.executeHookFunctions('nodeExecuteBefore', [nodeName]); + await additionalData.hooks?.runHook('nodeExecuteBefore', [nodeName]); } else { // Outputs taskData.executionTime = new Date().getTime() - taskData.startTime; - await additionalData.hooks?.executeHookFunctions('nodeExecuteAfter', [ + await additionalData.hooks?.runHook('nodeExecuteAfter', [ nodeName, taskData, this.runExecutionData, diff --git a/packages/core/src/node-execution-context/trigger-context.ts b/packages/core/src/execution-engine/node-execution-context/trigger-context.ts similarity index 93% rename from packages/core/src/node-execution-context/trigger-context.ts rename to packages/core/src/execution-engine/node-execution-context/trigger-context.ts index 88c8d91432..01bdaf6b03 100644 --- a/packages/core/src/node-execution-context/trigger-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/trigger-context.ts @@ -11,14 +11,14 @@ import { ApplicationError, createDeferredPromise } from 'n8n-workflow'; // eslint-disable-next-line import/no-cycle import { - getBinaryHelperFunctions, getRequestHelperFunctions, getSchedulingFunctions, getSSHTunnelFunctions, returnJsonArray, -} from '@/NodeExecuteFunctions'; +} from '@/node-execute-functions'; import { NodeExecutionContext } from './node-execution-context'; +import { getBinaryHelperFunctions } from './utils/binary-helper-functions'; const throwOnEmit = () => { throw new ApplicationError('Overwrite TriggerContext.emit function'); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/binary-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/binary-helper-functions.test.ts new file mode 100644 index 0000000000..a250421cdf --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/binary-helper-functions.test.ts @@ -0,0 +1,480 @@ +import { Container } from '@n8n/di'; +import { mkdtempSync, readFileSync } from 'fs'; +import { IncomingMessage } from 'http'; +import { mock } from 'jest-mock-extended'; +import type { + IBinaryData, + INode, + ITaskDataConnections, + IWorkflowExecuteAdditionalData, +} from 'n8n-workflow'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { Readable } from 'stream'; + +import { BinaryDataService } from '@/binary-data/binary-data.service'; + +import { + assertBinaryData, + binaryToString, + copyBinaryFile, + detectBinaryEncoding, + getBinaryDataBuffer, + getBinaryHelperFunctions, + prepareBinaryData, + setBinaryDataBuffer, +} from '../binary-helper-functions'; + +const workflowId = 'workflow123'; +const executionId = 'execution456'; + +const bufferToIncomingMessage = (buffer: Buffer, encoding = 'utf-8') => { + const incomingMessage = Readable.from(buffer) as IncomingMessage; + incomingMessage.headers = { 'content-type': `application/json;charset=${encoding}` }; + // @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks + incomingMessage.__proto__ = IncomingMessage.prototype; + return incomingMessage; +}; + +describe('test binary data helper methods', () => { + let binaryDataService: BinaryDataService; + const temporaryDir = mkdtempSync(join(tmpdir(), 'n8n')); + + beforeEach(() => { + binaryDataService = new BinaryDataService(); + Container.set(BinaryDataService, binaryDataService); + }); + + test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => { + // Setup a 'default' binary data manager instance + await binaryDataService.init({ + mode: 'default', + availableModes: ['default'], + localStoragePath: temporaryDir, + }); + + // Set our binary data buffer + const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); + const setBinaryDataBufferResponse: IBinaryData = await setBinaryDataBuffer( + { + mimeType: 'txt', + data: 'This should be overwritten by the actual payload in the response', + }, + inputData, + 'workflowId', + 'executionId', + ); + + // Expect our return object to contain the base64 encoding of the input data, as it should be stored in memory. + expect(setBinaryDataBufferResponse.data).toEqual(inputData.toString('base64')); + + // Now, re-fetch our data. + // An ITaskDataConnections object is used to share data between nodes. The top level property, 'main', represents the successful output object from a previous node. + const taskDataConnectionsInput: ITaskDataConnections = { + main: [], + }; + + // We add an input set, with one item at index 0, to this input. It contains an empty json payload and our binary data. + taskDataConnectionsInput.main.push([ + { + json: {}, + binary: { + data: setBinaryDataBufferResponse, + }, + }, + ]); + + // Now, lets fetch our data! The item will be item index 0. + const getBinaryDataBufferResponse: Buffer = await getBinaryDataBuffer( + taskDataConnectionsInput, + 0, + 'data', + 0, + ); + + expect(getBinaryDataBufferResponse).toEqual(inputData); + }); + + test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => { + // Setup a 'filesystem' binary data manager instance + await binaryDataService.init({ + mode: 'filesystem', + availableModes: ['filesystem'], + localStoragePath: temporaryDir, + }); + + // Set our binary data buffer + const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); + const setBinaryDataBufferResponse: IBinaryData = await setBinaryDataBuffer( + { + mimeType: 'txt', + data: 'This should be overwritten with the name of the configured data manager', + }, + inputData, + 'workflowId', + 'executionId', + ); + + // Expect our return object to contain the name of the configured data manager. + expect(setBinaryDataBufferResponse.data).toEqual('filesystem-v2'); + + // Ensure that the input data was successfully persisted to disk. + expect( + readFileSync( + `${temporaryDir}/${setBinaryDataBufferResponse.id?.replace('filesystem-v2:', '')}`, + ), + ).toEqual(inputData); + + // Now, re-fetch our data. + // An ITaskDataConnections object is used to share data between nodes. The top level property, 'main', represents the successful output object from a previous node. + const taskDataConnectionsInput: ITaskDataConnections = { + main: [], + }; + + // We add an input set, with one item at index 0, to this input. It contains an empty json payload and our binary data. + taskDataConnectionsInput.main.push([ + { + json: {}, + binary: { + data: setBinaryDataBufferResponse, + }, + }, + ]); + + // Now, lets fetch our data! The item will be item index 0. + const getBinaryDataBufferResponse: Buffer = await getBinaryDataBuffer( + taskDataConnectionsInput, + 0, + 'data', + 0, + ); + + expect(getBinaryDataBufferResponse).toEqual(inputData); + }); +}); + +describe('binaryToString', () => { + const ENCODING_SAMPLES = { + utf8: { + text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ', + buffer: Buffer.from([ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20, + 0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80, + 0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3, + 0xbc, 0x20, 0xc3, 0xb1, + ]), + }, + + 'iso-8859-15': { + text: 'Café € personnalité', + buffer: Buffer.from([ + 0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61, + 0x6c, 0x69, 0x74, 0xe9, + ]), + }, + + latin1: { + text: 'señor année déjà', + buffer: Buffer.from([ + 0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a, + 0xe0, + ]), + }, + + ascii: { + text: 'Hello, World! 123', + buffer: Buffer.from([ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31, + 0x32, 0x33, + ]), + }, + + 'windows-1252': { + text: '€ Smart "quotes" • bullet', + buffer: Buffer.from([ + 0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73, + 0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74, + ]), + }, + + 'shift-jis': { + text: 'こんにちは世界', + buffer: Buffer.from([ + 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45, + ]), + }, + + big5: { + text: '哈囉世界', + buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]), + }, + + 'koi8-r': { + text: 'Привет мир', + buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]), + }, + }; + + describe('should handle Buffer', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const data = await binaryToString(buffer, encoding); + expect(data).toBe(text); + }); + } + }); + + describe('should handle streams', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const stream = Readable.from(buffer); + const data = await binaryToString(stream, encoding); + expect(data).toBe(text); + }); + } + }); + + describe('should handle IncomingMessage', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const incomingMessage = bufferToIncomingMessage(buffer, encoding); + const data = await binaryToString(incomingMessage); + expect(data).toBe(text); + }); + } + }); + + it('should handle undefined encoding', async () => { + const buffer = Buffer.from('Test'); + const result = await binaryToString(buffer); + expect(result).toBe('Test'); + }); + + it('should handle stream with no explicit encoding', async () => { + const stream = Readable.from(Buffer.from('Test')); + const result = await binaryToString(stream); + expect(result).toBe('Test'); + }); +}); + +describe('detectBinaryEncoding', () => { + it('should detect encoding for utf-8 buffers', () => { + const utf8Buffer = Buffer.from('Hello, 世界'); + expect(detectBinaryEncoding(utf8Buffer)).toBe('UTF-8'); + }); + + it('should detect encoding for latin1 buffers', () => { + const latinBuffer = Buffer.from('señor', 'latin1'); + expect(detectBinaryEncoding(latinBuffer)).toBe('ISO-8859-1'); + }); + + it('should handle empty buffer', () => { + const emptyBuffer = Buffer.from(''); + expect(detectBinaryEncoding(emptyBuffer)).toBeDefined(); + }); +}); + +describe('assertBinaryData', () => { + const mockNode = mock({ name: 'Test Node' }); + + it('should throw error when no binary data exists', () => { + const inputData = { main: [[{ json: {} }]] }; + + expect(() => assertBinaryData(inputData, mockNode, 0, 'testFile', 0)).toThrow( + "expects the node's input data to contain a binary file", + ); + }); + + it('should throw error when specific binary property does not exist', () => { + const inputData = { + main: [ + [ + { + json: {}, + binary: { + otherFile: mock(), + }, + }, + ], + ], + }; + + expect(() => assertBinaryData(inputData, mockNode, 0, 'testFile', 0)).toThrow( + 'The item has no binary field', + ); + }); + + it('should return binary data when it exists', () => { + const binaryData = mock({ fileName: 'test.txt' }); + const inputData = { + main: [ + [ + { + json: {}, + binary: { + testFile: binaryData, + }, + }, + ], + ], + }; + + const result = assertBinaryData(inputData, mockNode, 0, 'testFile', 0); + expect(result).toBe(binaryData); + }); +}); + +describe('copyBinaryFile', () => { + const fileName = 'test.txt'; + const filePath = `/path/to/${fileName}`; + const binaryData: IBinaryData = { + data: '', + mimeType: 'text/plain', + fileName, + }; + + const binaryDataService = mock(); + + beforeEach(() => { + jest.resetAllMocks(); + Container.set(BinaryDataService, binaryDataService); + binaryDataService.copyBinaryFile.mockResolvedValueOnce(binaryData); + }); + + it('should handle files without explicit mime type', async () => { + const result = await copyBinaryFile(workflowId, executionId, filePath, fileName); + + expect(result.fileName).toBe(fileName); + expect(binaryDataService.copyBinaryFile).toHaveBeenCalledWith( + workflowId, + executionId, + { + ...binaryData, + fileExtension: 'txt', + fileType: 'text', + }, + filePath, + ); + }); + + it('should use provided mime type', async () => { + const result = await copyBinaryFile( + workflowId, + executionId, + filePath, + fileName, + 'application/octet-stream', + ); + + expect(result.fileName).toBe(fileName); + expect(binaryDataService.copyBinaryFile).toHaveBeenCalledWith( + workflowId, + executionId, + { + ...binaryData, + fileExtension: 'bin', + fileType: undefined, + mimeType: 'application/octet-stream', + }, + filePath, + ); + }); +}); + +describe('prepareBinaryData', () => { + const buffer: Buffer = Buffer.from('test', 'utf8'); + const binaryDataService = mock(); + + beforeEach(() => { + jest.resetAllMocks(); + Container.set(BinaryDataService, binaryDataService); + + binaryDataService.store.mockImplementation(async (_w, _e, _b, binaryData) => binaryData); + }); + + it('parses filenames correctly', async () => { + const fileName = 'test-file'; + + const result = await prepareBinaryData(buffer, executionId, workflowId, fileName); + + expect(result.fileName).toEqual(fileName); + expect(binaryDataService.store).toHaveBeenCalledWith(workflowId, executionId, buffer, { + data: '', + fileExtension: undefined, + fileName, + fileType: 'text', + mimeType: 'text/plain', + }); + }); + + it('handles IncomingMessage with responseUrl', async () => { + const incomingMessage = bufferToIncomingMessage(buffer); + incomingMessage.responseUrl = 'http://example.com/file.txt'; + + const result = await prepareBinaryData(incomingMessage, executionId, workflowId); + + expect(result.fileName).toBe('file.txt'); + expect(result.mimeType).toBe('text/plain'); + }); + + it('handles buffer with no detectable mime type', async () => { + const buffer = Buffer.from([0x00, 0x01, 0x02, 0x03]); + + const result = await prepareBinaryData(buffer, executionId, workflowId); + + expect(result.mimeType).toBe('text/plain'); + }); + + it('handles IncomingMessage with no content type or filename', async () => { + const incomingMessage = bufferToIncomingMessage(Buffer.from('test')); + delete incomingMessage.headers['content-type']; + delete incomingMessage.contentDisposition; + + const result = await prepareBinaryData(incomingMessage, executionId, workflowId); + + expect(result.mimeType).toBe('text/plain'); + }); +}); + +describe('setBinaryDataBuffer', () => { + it('should handle empty buffer', async () => { + const emptyBuffer = Buffer.from(''); + const binaryData: IBinaryData = { + mimeType: 'text/plain', + data: '', + }; + + const result = await setBinaryDataBuffer(binaryData, emptyBuffer, workflowId, executionId); + + expect(result).toBeDefined(); + expect(result.data).toBe(''); + }); +}); + +describe('getBinaryHelperFunctions', () => { + it('should return helper functions with correct context', async () => { + const additionalData = { executionId } as IWorkflowExecuteAdditionalData; + + const helperFunctions = getBinaryHelperFunctions(additionalData, workflowId); + + const expectedMethods = [ + 'getBinaryPath', + 'getBinaryStream', + 'getBinaryMetadata', + 'binaryToBuffer', + 'binaryToString', + 'prepareBinaryData', + 'setBinaryDataBuffer', + 'copyBinaryFile', + ] as const; + + expectedMethods.forEach((method) => { + expect(helperFunctions).toHaveProperty(method); + expect(typeof helperFunctions[method]).toBe('function'); + }); + + await expect(async () => await helperFunctions.copyBinaryFile()).rejects.toThrow( + '`copyBinaryFile` has been removed', + ); + }); +}); diff --git a/packages/core/src/node-execution-context/utils/__tests__/cleanupParameterData.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/cleanup-parameter-data.test.ts similarity index 94% rename from packages/core/src/node-execution-context/utils/__tests__/cleanupParameterData.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/cleanup-parameter-data.test.ts index 47913669b6..a3b22fcdd2 100644 --- a/packages/core/src/node-execution-context/utils/__tests__/cleanupParameterData.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/cleanup-parameter-data.test.ts @@ -2,7 +2,7 @@ import toPlainObject from 'lodash/toPlainObject'; import { DateTime } from 'luxon'; import type { NodeParameterValue } from 'n8n-workflow'; -import { cleanupParameterData } from '../cleanupParameterData'; +import { cleanupParameterData } from '../cleanup-parameter-data'; describe('cleanupParameterData', () => { it('should stringify Luxon dates in-place', () => { diff --git a/packages/core/test/CreateNodeAsTool.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/create-node-as-tool.test.ts similarity index 99% rename from packages/core/test/CreateNodeAsTool.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/create-node-as-tool.test.ts index fdc14269e1..98827bce61 100644 --- a/packages/core/test/CreateNodeAsTool.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/create-node-as-tool.test.ts @@ -2,7 +2,7 @@ import { mock } from 'jest-mock-extended'; import type { INodeType, ISupplyDataFunctions, INode } from 'n8n-workflow'; import { z } from 'zod'; -import { createNodeAsTool } from '@/CreateNodeAsTool'; +import { createNodeAsTool } from '../create-node-as-tool'; jest.mock('@langchain/core/tools', () => ({ DynamicStructuredTool: jest.fn().mockImplementation((config) => ({ @@ -319,7 +319,7 @@ describe('createNodeAsTool', () => { const tool = createNodeAsTool(options).response; - expect(tool.schema.shape.complexJson._def.innerType).toBeInstanceOf(z.ZodRecord); + expect(tool.schema.shape.complexJson._def.innerType).toBeInstanceOf(z.ZodEffects); expect(tool.schema.shape.complexJson.description).toBe('Param with complex JSON default'); expect(tool.schema.shape.complexJson._def.defaultValue()).toEqual({ nested: { key: 'value' }, diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/deduplication-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/deduplication-helper-functions.test.ts new file mode 100644 index 0000000000..a9bfe54b90 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/deduplication-helper-functions.test.ts @@ -0,0 +1,25 @@ +import { mock } from 'jest-mock-extended'; +import type { Workflow, INode } from 'n8n-workflow'; + +import { getDeduplicationHelperFunctions } from '../deduplication-helper-functions'; + +describe('getDeduplicationHelperFunctions', () => { + const workflow = mock(); + const node = mock(); + const helperFunctions = getDeduplicationHelperFunctions(workflow, node); + + it('should create helper functions with correct context', () => { + const expectedMethods = [ + 'checkProcessedAndRecord', + 'checkProcessedItemsAndRecord', + 'removeProcessed', + 'clearAllProcessedItems', + 'getProcessedDataCount', + ] as const; + + expectedMethods.forEach((method) => { + expect(helperFunctions).toHaveProperty(method); + expect(typeof helperFunctions[method]).toBe('function'); + }); + }); +}); diff --git a/packages/core/src/node-execution-context/utils/__tests__/ensureType.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/ensure-type.test.ts similarity index 98% rename from packages/core/src/node-execution-context/utils/__tests__/ensureType.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/ensure-type.test.ts index 1637d988c9..374b3d43bc 100644 --- a/packages/core/src/node-execution-context/utils/__tests__/ensureType.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/ensure-type.test.ts @@ -1,6 +1,6 @@ import { ExpressionError } from 'n8n-workflow'; -import { ensureType } from '../ensureType'; +import { ensureType } from '../ensure-type'; describe('ensureType', () => { it('throws error for null value', () => { diff --git a/packages/core/test/WorkflowExecutionMetadata.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/execution-metadata.test.ts similarity index 88% rename from packages/core/test/WorkflowExecutionMetadata.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/execution-metadata.test.ts index 63d0892e6a..b08e38da5b 100644 --- a/packages/core/test/WorkflowExecutionMetadata.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/execution-metadata.test.ts @@ -1,13 +1,14 @@ import type { IRunExecutionData } from 'n8n-workflow'; import { InvalidExecutionMetadataError } from '@/errors/invalid-execution-metadata.error'; + import { setWorkflowExecutionMetadata, setAllWorkflowExecutionMetadata, KV_LIMIT, getWorkflowExecutionMetadata, getAllWorkflowExecutionMetadata, -} from '@/ExecutionMetadata'; +} from '../execution-metadata'; describe('Execution Metadata functions', () => { test('setWorkflowExecutionMetadata will set a value', () => { @@ -204,15 +205,12 @@ describe('Execution Metadata functions', () => { }, } as IRunExecutionData; - setWorkflowExecutionMetadata( - executionData, - 'test1', - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab', - ); + const longValue = 'a'.repeat(513); + + setWorkflowExecutionMetadata(executionData, 'test1', longValue); expect(metadata).toEqual({ - test1: - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + test1: longValue.slice(0, 512), }); }); }); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/file-system-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/file-system-helper-functions.test.ts new file mode 100644 index 0000000000..d031fb4993 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/file-system-helper-functions.test.ts @@ -0,0 +1,186 @@ +import { Container } from '@n8n/di'; +import type { INode } from 'n8n-workflow'; +import { createReadStream } from 'node:fs'; +import { access as fsAccess } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { + BINARY_DATA_STORAGE_PATH, + BLOCK_FILE_ACCESS_TO_N8N_FILES, + CONFIG_FILES, + CUSTOM_EXTENSION_ENV, + RESTRICT_FILE_ACCESS_TO, + UM_EMAIL_TEMPLATES_INVITE, + UM_EMAIL_TEMPLATES_PWRESET, +} from '@/constants'; +import { InstanceSettings } from '@/instance-settings'; + +import { getFileSystemHelperFunctions, isFilePathBlocked } from '../file-system-helper-functions'; + +jest.mock('node:fs'); +jest.mock('node:fs/promises'); + +const originalProcessEnv = { ...process.env }; + +let instanceSettings: InstanceSettings; +beforeEach(() => { + process.env = { ...originalProcessEnv }; + + const error = new Error('ENOENT'); + // @ts-expect-error undefined property + error.code = 'ENOENT'; + (fsAccess as jest.Mock).mockRejectedValue(error); + + instanceSettings = Container.get(InstanceSettings); +}); + +describe('isFilePathBlocked', () => { + beforeEach(() => { + process.env[BLOCK_FILE_ACCESS_TO_N8N_FILES] = 'true'; + }); + + it('should return true for static cache dir', () => { + const filePath = instanceSettings.staticCacheDir; + expect(isFilePathBlocked(filePath)).toBe(true); + }); + + it('should return true for restricted paths', () => { + const restrictedPath = instanceSettings.n8nFolder; + expect(isFilePathBlocked(restrictedPath)).toBe(true); + }); + + it('should handle empty allowed paths', () => { + delete process.env[RESTRICT_FILE_ACCESS_TO]; + const result = isFilePathBlocked('/some/random/path'); + expect(result).toBe(false); + }); + + it('should handle multiple allowed paths', () => { + process.env[RESTRICT_FILE_ACCESS_TO] = '/path1;/path2;/path3'; + const allowedPath = '/path2/somefile'; + expect(isFilePathBlocked(allowedPath)).toBe(false); + }); + + it('should handle empty strings in allowed paths', () => { + process.env[RESTRICT_FILE_ACCESS_TO] = '/path1;;/path2'; + const allowedPath = '/path2/somefile'; + expect(isFilePathBlocked(allowedPath)).toBe(false); + }); + + it('should trim whitespace in allowed paths', () => { + process.env[RESTRICT_FILE_ACCESS_TO] = ' /path1 ; /path2 ; /path3 '; + const allowedPath = '/path2/somefile'; + expect(isFilePathBlocked(allowedPath)).toBe(false); + }); + + it('should return false when BLOCK_FILE_ACCESS_TO_N8N_FILES is false', () => { + process.env[BLOCK_FILE_ACCESS_TO_N8N_FILES] = 'false'; + const restrictedPath = instanceSettings.n8nFolder; + expect(isFilePathBlocked(restrictedPath)).toBe(false); + }); + + it('should return true when path is in allowed paths but still restricted', () => { + process.env[RESTRICT_FILE_ACCESS_TO] = '/some/allowed/path'; + const restrictedPath = instanceSettings.n8nFolder; + expect(isFilePathBlocked(restrictedPath)).toBe(true); + }); + + it('should return false when path is in allowed paths', () => { + const allowedPath = '/some/allowed/path'; + process.env[RESTRICT_FILE_ACCESS_TO] = allowedPath; + expect(isFilePathBlocked(allowedPath)).toBe(false); + }); + + it('should return true when file paths in CONFIG_FILES', () => { + process.env[CONFIG_FILES] = '/path/to/config1,/path/to/config2'; + const configPath = '/path/to/config1/somefile'; + expect(isFilePathBlocked(configPath)).toBe(true); + }); + + it('should return true when file paths in CUSTOM_EXTENSION_ENV', () => { + process.env[CUSTOM_EXTENSION_ENV] = '/path/to/extensions1;/path/to/extensions2'; + const extensionPath = '/path/to/extensions1/somefile'; + expect(isFilePathBlocked(extensionPath)).toBe(true); + }); + + it('should return true when file paths in BINARY_DATA_STORAGE_PATH', () => { + process.env[BINARY_DATA_STORAGE_PATH] = '/path/to/binary/storage'; + const binaryPath = '/path/to/binary/storage/somefile'; + expect(isFilePathBlocked(binaryPath)).toBe(true); + }); + + it('should block file paths in email template paths', () => { + process.env[UM_EMAIL_TEMPLATES_INVITE] = '/path/to/invite/templates'; + process.env[UM_EMAIL_TEMPLATES_PWRESET] = '/path/to/pwreset/templates'; + + const invitePath = '/path/to/invite/templates/invite.html'; + const pwResetPath = '/path/to/pwreset/templates/reset.html'; + + expect(isFilePathBlocked(invitePath)).toBe(true); + expect(isFilePathBlocked(pwResetPath)).toBe(true); + }); +}); + +describe('getFileSystemHelperFunctions', () => { + const node = { type: 'TestNode' } as INode; + const helperFunctions = getFileSystemHelperFunctions(node); + + it('should create helper functions with correct context', () => { + const expectedMethods = ['createReadStream', 'getStoragePath', 'writeContentToFile'] as const; + + expectedMethods.forEach((method) => { + expect(helperFunctions).toHaveProperty(method); + expect(typeof helperFunctions[method]).toBe('function'); + }); + }); + + describe('getStoragePath', () => { + it('returns correct path', () => { + const expectedPath = join(instanceSettings.n8nFolder, `storage/${node.type}`); + expect(helperFunctions.getStoragePath()).toBe(expectedPath); + }); + }); + + describe('createReadStream', () => { + it('should throw error for non-existent file', async () => { + const filePath = '/non/existent/file'; + const error = new Error('ENOENT'); + // @ts-expect-error undefined property + error.code = 'ENOENT'; + (fsAccess as jest.Mock).mockRejectedValueOnce(error); + + await expect(helperFunctions.createReadStream(filePath)).rejects.toThrow( + `The file "${filePath}" could not be accessed.`, + ); + }); + + it('should throw when file access is blocked', async () => { + process.env[RESTRICT_FILE_ACCESS_TO] = '/allowed/path'; + (fsAccess as jest.Mock).mockResolvedValueOnce({}); + await expect(helperFunctions.createReadStream('/blocked/path')).rejects.toThrow( + 'Access to the file is not allowed', + ); + }); + + it('should create a read stream if file access is permitted', async () => { + const filePath = '/allowed/path'; + (fsAccess as jest.Mock).mockResolvedValueOnce({}); + await helperFunctions.createReadStream(filePath); + expect(createReadStream).toHaveBeenCalledWith(filePath); + }); + }); + + describe('writeContentToFile', () => { + it('should throw error for blocked file path', async () => { + process.env[BLOCK_FILE_ACCESS_TO_N8N_FILES] = 'true'; + + await expect( + helperFunctions.writeContentToFile( + instanceSettings.n8nFolder + '/test.txt', + 'content', + 'w', + ), + ).rejects.toThrow('not writable'); + }); + }); +}); diff --git a/packages/core/src/node-execution-context/utils/__tests__/getAdditionalKeys.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/get-additional-keys.test.ts similarity index 97% rename from packages/core/src/node-execution-context/utils/__tests__/getAdditionalKeys.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/get-additional-keys.test.ts index 6ac1fbdc07..b78de39a6d 100644 --- a/packages/core/src/node-execution-context/utils/__tests__/getAdditionalKeys.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/get-additional-keys.test.ts @@ -7,9 +7,9 @@ import type { SecretsHelpersBase, } from 'n8n-workflow'; -import { PLACEHOLDER_EMPTY_EXECUTION_ID } from '@/Constants'; +import { PLACEHOLDER_EMPTY_EXECUTION_ID } from '@/constants'; -import { getAdditionalKeys } from '../getAdditionalKeys'; +import { getAdditionalKeys } from '../get-additional-keys'; describe('getAdditionalKeys', () => { const secretsHelpers = mock(); diff --git a/packages/core/src/node-execution-context/utils/__tests__/getInputConnectionData.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/get-input-connection-data.test.ts similarity index 100% rename from packages/core/src/node-execution-context/utils/__tests__/getInputConnectionData.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/get-input-connection-data.test.ts diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-incoming-message.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-incoming-message.test.ts new file mode 100644 index 0000000000..0c2ca3113e --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-incoming-message.test.ts @@ -0,0 +1,252 @@ +import type { IncomingMessage } from 'http'; +import { mock } from 'jest-mock-extended'; + +import { + parseContentDisposition, + parseContentType, + parseIncomingMessage, +} from '../parse-incoming-message'; + +describe('parseContentType', () => { + const testCases = [ + { + input: 'text/plain', + expected: { + type: 'text/plain', + parameters: { + charset: 'utf-8', + }, + }, + description: 'should parse basic content type', + }, + { + input: 'TEXT/PLAIN', + expected: { + type: 'text/plain', + parameters: { + charset: 'utf-8', + }, + }, + description: 'should convert type to lowercase', + }, + { + input: 'text/html; charset=iso-8859-1', + expected: { + type: 'text/html', + parameters: { + charset: 'iso-8859-1', + }, + }, + description: 'should parse content type with charset', + }, + { + input: 'application/json; charset=utf-8; boundary=---123', + expected: { + type: 'application/json', + parameters: { + charset: 'utf-8', + boundary: '---123', + }, + }, + description: 'should parse content type with multiple parameters', + }, + { + input: 'text/plain; charset="utf-8"; filename="test.txt"', + expected: { + type: 'text/plain', + parameters: { + charset: 'utf-8', + filename: 'test.txt', + }, + }, + description: 'should handle quoted parameter values', + }, + { + input: 'text/plain; filename=%22test%20file.txt%22', + expected: { + type: 'text/plain', + parameters: { + charset: 'utf-8', + filename: 'test file.txt', + }, + }, + description: 'should handle encoded parameter values', + }, + { + input: undefined, + expected: null, + description: 'should return null for undefined input', + }, + { + input: '', + expected: null, + description: 'should return null for empty string', + }, + ]; + + test.each(testCases)('$description', ({ input, expected }) => { + expect(parseContentType(input)).toEqual(expected); + }); +}); + +describe('parseContentDisposition', () => { + const testCases = [ + { + input: 'attachment; filename="file.txt"', + expected: { type: 'attachment', filename: 'file.txt' }, + description: 'should parse basic content disposition', + }, + { + input: 'attachment; filename=file.txt', + expected: { type: 'attachment', filename: 'file.txt' }, + description: 'should parse filename without quotes', + }, + { + input: 'inline; filename="image.jpg"', + expected: { type: 'inline', filename: 'image.jpg' }, + description: 'should parse inline disposition', + }, + { + input: 'attachment; filename="my file.pdf"', + expected: { type: 'attachment', filename: 'my file.pdf' }, + description: 'should parse filename with spaces', + }, + { + input: "attachment; filename*=UTF-8''my%20file.txt", + expected: { type: 'attachment', filename: 'my file.txt' }, + description: 'should parse filename* parameter (RFC 5987)', + }, + { + input: 'filename="test.txt"', + expected: { type: 'attachment', filename: 'test.txt' }, + description: 'should handle invalid syntax but with filename', + }, + { + input: 'filename=test.txt', + expected: { type: 'attachment', filename: 'test.txt' }, + description: 'should handle invalid syntax with only filename parameter', + }, + { + input: undefined, + expected: null, + description: 'should return null for undefined input', + }, + { + input: '', + expected: null, + description: 'should return null for empty string', + }, + { + input: 'attachment; filename="%F0%9F%98%80.txt"', + expected: { type: 'attachment', filename: '😀.txt' }, + description: 'should handle encoded filenames', + }, + { + input: 'attachment; size=123; filename="test.txt"; creation-date="Thu, 1 Jan 2020"', + expected: { type: 'attachment', filename: 'test.txt' }, + description: 'should handle multiple parameters', + }, + ]; + + test.each(testCases)('$description', ({ input, expected }) => { + expect(parseContentDisposition(input)).toEqual(expected); + }); +}); + +describe('parseIncomingMessage', () => { + it('parses valid content-type header', () => { + const message = mock({ + headers: { 'content-type': 'application/json', 'content-disposition': undefined }, + }); + parseIncomingMessage(message); + + expect(message.contentType).toEqual('application/json'); + }); + + it('parses valid content-type header with parameters', () => { + const message = mock({ + headers: { + 'content-type': 'application/json; charset=utf-8', + 'content-disposition': undefined, + }, + }); + parseIncomingMessage(message); + + expect(message.contentType).toEqual('application/json'); + expect(message.encoding).toEqual('utf-8'); + }); + + it('parses valid content-type header with encoding wrapped in quotes', () => { + const message = mock({ + headers: { + 'content-type': 'application/json; charset="utf-8"', + 'content-disposition': undefined, + }, + }); + parseIncomingMessage(message); + + expect(message.contentType).toEqual('application/json'); + expect(message.encoding).toEqual('utf-8'); + }); + + it('parses valid content-disposition header with filename*', () => { + const message = mock({ + headers: { + 'content-type': undefined, + 'content-disposition': + 'attachment; filename="screenshot%20(1).png"; filename*=UTF-8\'\'screenshot%20(1).png', + }, + }); + parseIncomingMessage(message); + + expect(message.contentDisposition).toEqual({ + filename: 'screenshot (1).png', + type: 'attachment', + }); + }); + + it('parses valid content-disposition header with filename* (quoted)', () => { + const message = mock({ + headers: { + 'content-type': undefined, + 'content-disposition': ' attachment;filename*="utf-8\' \'test-unsplash.jpg"', + }, + }); + parseIncomingMessage(message); + + expect(message.contentDisposition).toEqual({ + filename: 'test-unsplash.jpg', + type: 'attachment', + }); + }); + + it('parses valid content-disposition header with filename and trailing ";"', () => { + const message = mock({ + headers: { + 'content-type': undefined, + 'content-disposition': 'inline; filename="screenshot%20(1).png";', + }, + }); + parseIncomingMessage(message); + + expect(message.contentDisposition).toEqual({ + filename: 'screenshot (1).png', + type: 'inline', + }); + }); + + it('parses non standard content-disposition with missing type', () => { + const message = mock({ + headers: { + 'content-type': undefined, + 'content-disposition': 'filename="screenshot%20(1).png";', + }, + }); + parseIncomingMessage(message); + + expect(message.contentDisposition).toEqual({ + filename: 'screenshot (1).png', + type: 'attachment', + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts new file mode 100644 index 0000000000..9812057a4b --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts @@ -0,0 +1,133 @@ +import FormData from 'form-data'; +import type { Agent } from 'https'; +import { mock } from 'jest-mock-extended'; +import type { IHttpRequestMethods, IRequestOptions } from 'n8n-workflow'; +import type { SecureContextOptions } from 'tls'; + +import { parseRequestObject } from '../parse-request-object'; + +describe('parseRequestObject', () => { + test('should handle basic request options', async () => { + const axiosOptions = await parseRequestObject({ + url: 'https://example.com', + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: { key: 'value' }, + }); + + expect(axiosOptions).toEqual( + expect.objectContaining({ + url: 'https://example.com', + method: 'POST', + headers: { accept: '*/*', 'content-type': 'application/json' }, + data: { key: 'value' }, + maxRedirects: 0, + }), + ); + }); + + test('should set correct headers for FormData', async () => { + const formData = new FormData(); + formData.append('key', 'value'); + + const axiosOptions = await parseRequestObject({ + url: 'https://example.com', + formData, + headers: { + 'content-type': 'multipart/form-data', + }, + }); + + expect(axiosOptions.headers).toMatchObject({ + accept: '*/*', + 'content-length': 163, + 'content-type': expect.stringMatching(/^multipart\/form-data; boundary=/), + }); + + expect(axiosOptions.data).toBeInstanceOf(FormData); + }); + + test('should not use Host header for SNI', async () => { + const axiosOptions = await parseRequestObject({ + url: 'https://example.de/foo/bar', + headers: { Host: 'other.host.com' }, + }); + expect((axiosOptions.httpsAgent as Agent).options.servername).toEqual('example.de'); + }); + + describe('should set SSL certificates', () => { + const agentOptions: SecureContextOptions = { + ca: '-----BEGIN CERTIFICATE-----\nTEST\n-----END CERTIFICATE-----', + }; + const requestObject: IRequestOptions = { + method: 'GET', + uri: 'https://example.de', + agentOptions, + }; + + test('on regular requests', async () => { + const axiosOptions = await parseRequestObject(requestObject); + expect((axiosOptions.httpsAgent as Agent).options).toEqual({ + servername: 'example.de', + ...agentOptions, + noDelay: true, + path: null, + }); + }); + + test('on redirected requests', async () => { + const axiosOptions = await parseRequestObject(requestObject); + expect(axiosOptions.beforeRedirect).toBeDefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const redirectOptions: Record = { agents: {}, hostname: 'example.de' }; + axiosOptions.beforeRedirect!(redirectOptions, mock()); + expect(redirectOptions.agent).toEqual(redirectOptions.agents.https); + expect((redirectOptions.agent as Agent).options).toEqual({ + servername: 'example.de', + ...agentOptions, + noDelay: true, + path: null, + }); + }); + }); + + describe('when followRedirect is true', () => { + test.each(['GET', 'HEAD'] as IHttpRequestMethods[])( + 'should set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followRedirect: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(1234); + }, + ); + + test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'should not set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followRedirect: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(0); + }, + ); + }); + + describe('when followAllRedirects is true', () => { + test.each(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'should set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followAllRedirects: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(1234); + }, + ); + }); +}); diff --git a/packages/core/src/node-execution-context/utils/__tests__/validateValueAgainstSchema.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/validate-value-against-schema.test.ts similarity index 53% rename from packages/core/src/node-execution-context/utils/__tests__/validateValueAgainstSchema.test.ts rename to packages/core/src/execution-engine/node-execution-context/utils/__tests__/validate-value-against-schema.test.ts index e09299a457..60d4ca5367 100644 --- a/packages/core/src/node-execution-context/utils/__tests__/validateValueAgainstSchema.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/validate-value-against-schema.test.ts @@ -1,6 +1,6 @@ -import type { IDataObject, INode, INodeType } from 'n8n-workflow'; +import { ExpressionError, type IDataObject, type INode, type INodeType } from 'n8n-workflow'; -import { validateValueAgainstSchema } from '../validateValueAgainstSchema'; +import { validateValueAgainstSchema } from '../validate-value-against-schema'; describe('validateValueAgainstSchema', () => { test('should validate fixedCollection values parameter', () => { @@ -246,4 +246,220 @@ describe('validateValueAgainstSchema', () => { // value should be type number expect(typeof result).toEqual('number'); }); + + describe('when validating a resource mapper value', () => { + describe('when attemptToConvertTypes === true', () => { + const nodeType = { + description: { + properties: [ + { + name: 'operation', + type: 'resourceMapper', + typeOptions: { + resourceMapper: { + mode: 'add', + }, + }, + }, + ], + }, + } as unknown as INodeType; + + const node = { + parameters: { + operation: { + schema: [ + { id: 'num', type: 'number', required: true }, + { id: 'str', type: 'string', required: true }, + { id: 'obj', type: 'object', required: true }, + { id: 'arr', type: 'array', required: true }, + ], + attemptToConvertTypes: true, + mappingMode: '', + value: '', + }, + }, + } as unknown as INode; + + const parameterName = 'operation.value'; + + describe('should correctly validate values for', () => { + test.each([ + { num: 0 }, + { num: 23 }, + { num: -0 }, + { num: -Infinity }, + { num: Infinity }, + { str: '' }, + { str: ' ' }, + { str: 'hello' }, + { arr: [] }, + { obj: {} }, + ])('%s', (value) => { + expect(() => + validateValueAgainstSchema(node, nodeType, value, parameterName, 0, 0), + ).not.toThrow(); + }); + }); + + describe('should throw an error for', () => { + test.each([{ num: NaN }, { num: undefined }, { num: null }])('%s', (value) => { + expect(() => + validateValueAgainstSchema(node, nodeType, value, parameterName, 0, 0), + ).toThrow(); + }); + }); + }); + + describe('when showTypeConversionOptions is not set (=default)', () => { + test('should correctly convert types', () => { + const nodeType = { + description: { + properties: [ + { + displayName: 'Columns', + name: 'columns', + type: 'resourceMapper', + required: true, + typeOptions: { + loadOptionsDependsOn: ['table.value', 'operation'], + resourceMapper: { + mode: 'upsert', + }, + }, + }, + ], + }, + } as unknown as INodeType; + + const node: INode = { + parameters: { + columns: { + mappingMode: 'defineBelow', + value: { + id: 2, + count: '={{ $json.count }}', + }, + matchingColumns: ['id'], + attemptToConvertTypes: false, + convertFieldsToString: true, + schema: [ + { + id: 'id', + displayName: 'id', + required: false, + defaultMatch: true, + display: true, + type: 'number', + canBeUsedToMatch: true, + }, + { + id: 'count', + displayName: 'count', + required: false, + defaultMatch: false, + display: true, + type: 'number', + canBeUsedToMatch: false, + }, + ], + }, + options: {}, + }, + id: '8d6cec63-8db1-440c-8966-4d6311ee69a9', + name: 'add products to DB', + type: 'n8n-nodes-base.postgres', + typeVersion: 2.3, + position: [420, 0], + }; + + const value = { + id: 2, + count: '23', + }; + + const parameterName = 'columns.value'; + + const result = validateValueAgainstSchema(node, nodeType, value, parameterName, 0, 0); + + expect(result).toEqual({ + id: 2, + count: 23, + }); + }); + }); + + describe('when showTypeConversionOptions is true', () => { + test('should throw an error', () => { + const nodeType = { + description: { + properties: [ + { + displayName: 'Columns', + name: 'columns', + type: 'resourceMapper', + noDataExpression: true, + typeOptions: { + resourceMapper: { + showTypeConversionOptions: true, + mode: 'upsert', + }, + }, + }, + ], + }, + } as unknown as INodeType; + + const node: INode = { + parameters: { + columns: { + mappingMode: 'defineBelow', + value: { + id: 2, + count: '={{ $json.count }}', + }, + matchingColumns: ['id'], + schema: [ + { + id: 'id', + displayName: 'id', + required: false, + defaultMatch: true, + display: true, + type: 'number', + canBeUsedToMatch: true, + }, + { + id: 'count', + displayName: 'count', + required: false, + defaultMatch: false, + display: true, + type: 'number', + canBeUsedToMatch: false, + }, + ], + }, + options: {}, + }, + id: '8d6cec63-8db1-440c-8966-4d6311ee69a9', + name: 'add products to DB', + type: 'n8n-nodes-base.postgres', + typeVersion: 2.3, + position: [420, 0], + }; + + const value = { + id: 2, + count: '23', + }; + + const parameterName = 'columns.value'; + + expect(() => + validateValueAgainstSchema(node, nodeType, value, parameterName, 0, 0), + ).toThrow(new ExpressionError("Invalid input for 'count' [item 0]")); + }); + }); + }); }); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/binary-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/binary-helper-functions.ts new file mode 100644 index 0000000000..c1bbff7899 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/binary-helper-functions.ts @@ -0,0 +1,289 @@ +import { Container } from '@n8n/di'; +import chardet from 'chardet'; +import FileType from 'file-type'; +import { IncomingMessage } from 'http'; +import iconv from 'iconv-lite'; +import { extension, lookup } from 'mime-types'; +import type { + BinaryHelperFunctions, + IBinaryData, + INode, + ITaskDataConnections, + IWorkflowExecuteAdditionalData, +} from 'n8n-workflow'; +import { NodeOperationError, fileTypeFromMimeType, ApplicationError } from 'n8n-workflow'; +import path from 'path'; +import type { Readable } from 'stream'; +import { URL } from 'url'; + +import { BinaryDataService } from '@/binary-data/binary-data.service'; +import type { BinaryData } from '@/binary-data/types'; +import { binaryToBuffer } from '@/binary-data/utils'; + +import { parseIncomingMessage } from './parse-incoming-message'; + +export async function binaryToString(body: Buffer | Readable, encoding?: string) { + if (!encoding && body instanceof IncomingMessage) { + parseIncomingMessage(body); + encoding = body.encoding; + } + const buffer = await binaryToBuffer(body); + return iconv.decode(buffer, encoding ?? 'utf-8'); +} + +function getBinaryPath(binaryDataId: string): string { + return Container.get(BinaryDataService).getPath(binaryDataId); +} + +/** + * Returns binary file metadata + */ +async function getBinaryMetadata(binaryDataId: string): Promise { + return await Container.get(BinaryDataService).getMetadata(binaryDataId); +} + +/** + * Returns binary file stream for piping + */ +async function getBinaryStream(binaryDataId: string, chunkSize?: number): Promise { + return await Container.get(BinaryDataService).getAsStream(binaryDataId, chunkSize); +} + +export function assertBinaryData( + inputData: ITaskDataConnections, + node: INode, + itemIndex: number, + propertyName: string, + inputIndex: number, +): IBinaryData { + const binaryKeyData = inputData.main[inputIndex]![itemIndex].binary; + if (binaryKeyData === undefined) { + throw new NodeOperationError( + node, + `This operation expects the node's input data to contain a binary file '${propertyName}', but none was found [item ${itemIndex}]`, + { + itemIndex, + description: 'Make sure that the previous node outputs a binary file', + }, + ); + } + + const binaryPropertyData = binaryKeyData[propertyName]; + if (binaryPropertyData === undefined) { + throw new NodeOperationError( + node, + `The item has no binary field '${propertyName}' [item ${itemIndex}]`, + { + itemIndex, + description: + 'Check that the parameter where you specified the input binary field name is correct, and that it matches a field in the binary input', + }, + ); + } + + return binaryPropertyData; +} + +/** + * Returns binary data buffer for given item index and property name. + */ +export async function getBinaryDataBuffer( + inputData: ITaskDataConnections, + itemIndex: number, + propertyName: string, + inputIndex: number, +): Promise { + const binaryData = inputData.main[inputIndex]![itemIndex].binary![propertyName]; + return await Container.get(BinaryDataService).getAsBuffer(binaryData); +} + +export function detectBinaryEncoding(buffer: Buffer): string { + return chardet.detect(buffer) as string; +} + +/** + * Store an incoming IBinaryData & related buffer using the configured binary data manager. + * + * @export + * @param {IBinaryData} binaryData + * @param {Buffer | Readable} bufferOrStream + * @returns {Promise} + */ +export async function setBinaryDataBuffer( + binaryData: IBinaryData, + bufferOrStream: Buffer | Readable, + workflowId: string, + executionId: string, +): Promise { + return await Container.get(BinaryDataService).store( + workflowId, + executionId, + bufferOrStream, + binaryData, + ); +} + +export async function copyBinaryFile( + workflowId: string, + executionId: string, + filePath: string, + fileName: string, + mimeType?: string, +): Promise { + let fileExtension: string | undefined; + if (!mimeType) { + // If no mime type is given figure it out + + if (filePath) { + // Use file path to guess mime type + const mimeTypeLookup = lookup(filePath); + if (mimeTypeLookup) { + mimeType = mimeTypeLookup; + } + } + + if (!mimeType) { + // read the first bytes of the file to guess mime type + const fileTypeData = await FileType.fromFile(filePath); + if (fileTypeData) { + mimeType = fileTypeData.mime; + fileExtension = fileTypeData.ext; + } + } + } + + if (!fileExtension && mimeType) { + fileExtension = extension(mimeType) || undefined; + } + + if (!mimeType) { + // Fall back to text + mimeType = 'text/plain'; + } + + const returnData: IBinaryData = { + mimeType, + fileType: fileTypeFromMimeType(mimeType), + fileExtension, + data: '', + }; + + if (fileName) { + returnData.fileName = fileName; + } else if (filePath) { + returnData.fileName = path.parse(filePath).base; + } + + return await Container.get(BinaryDataService).copyBinaryFile( + workflowId, + executionId, + returnData, + filePath, + ); +} + +/** + * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as + * base64 and adds metadata. + */ +// eslint-disable-next-line complexity +export async function prepareBinaryData( + binaryData: Buffer | Readable, + executionId: string, + workflowId: string, + filePath?: string, + mimeType?: string, +): Promise { + let fileExtension: string | undefined; + if (binaryData instanceof IncomingMessage) { + if (!filePath) { + try { + const { responseUrl } = binaryData; + filePath = + binaryData.contentDisposition?.filename ?? + ((responseUrl && new URL(responseUrl).pathname) ?? binaryData.req?.path)?.slice(1); + } catch {} + } + if (!mimeType) { + mimeType = binaryData.contentType; + } + } + + if (!mimeType) { + // If no mime type is given figure it out + + if (filePath) { + // Use file path to guess mime type + const mimeTypeLookup = lookup(filePath); + if (mimeTypeLookup) { + mimeType = mimeTypeLookup; + } + } + + if (!mimeType) { + if (Buffer.isBuffer(binaryData)) { + // Use buffer to guess mime type + const fileTypeData = await FileType.fromBuffer(binaryData); + if (fileTypeData) { + mimeType = fileTypeData.mime; + fileExtension = fileTypeData.ext; + } + } else if (binaryData instanceof IncomingMessage) { + mimeType = binaryData.headers['content-type']; + } else { + // TODO: detect filetype from other kind of streams + } + } + } + + if (!fileExtension && mimeType) { + fileExtension = extension(mimeType) || undefined; + } + + if (!mimeType) { + // Fall back to text + mimeType = 'text/plain'; + } + + const returnData: IBinaryData = { + mimeType, + fileType: fileTypeFromMimeType(mimeType), + fileExtension, + data: '', + }; + + if (filePath) { + const filePathParts = path.parse(filePath); + + if (filePathParts.dir !== '') { + returnData.directory = filePathParts.dir; + } + returnData.fileName = filePathParts.base; + + // Remove the dot + fileExtension = filePathParts.ext.slice(1); + if (fileExtension) { + returnData.fileExtension = fileExtension; + } + } + + return await setBinaryDataBuffer(returnData, binaryData, workflowId, executionId); +} + +export const getBinaryHelperFunctions = ( + { executionId }: IWorkflowExecuteAdditionalData, + workflowId: string, +): BinaryHelperFunctions => ({ + getBinaryPath, + getBinaryStream, + getBinaryMetadata, + binaryToBuffer, + binaryToString, + prepareBinaryData: async (binaryData, filePath, mimeType) => + await prepareBinaryData(binaryData, executionId!, workflowId, filePath, mimeType), + setBinaryDataBuffer: async (data, binaryData) => + await setBinaryDataBuffer(data, binaryData, workflowId, executionId!), + copyBinaryFile: async () => { + throw new ApplicationError('`copyBinaryFile` has been removed. Please upgrade this node.'); + }, +}); diff --git a/packages/core/src/node-execution-context/utils/cleanupParameterData.ts b/packages/core/src/execution-engine/node-execution-context/utils/cleanup-parameter-data.ts similarity index 100% rename from packages/core/src/node-execution-context/utils/cleanupParameterData.ts rename to packages/core/src/execution-engine/node-execution-context/utils/cleanup-parameter-data.ts diff --git a/packages/core/src/execution-engine/node-execution-context/utils/create-node-as-tool.ts b/packages/core/src/execution-engine/node-execution-context/utils/create-node-as-tool.ts new file mode 100644 index 0000000000..9f1e707a67 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/create-node-as-tool.ts @@ -0,0 +1,140 @@ +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { generateZodSchema, NodeOperationError, traverseNodeParameters } from 'n8n-workflow'; +import type { IDataObject, INode, INodeType, FromAIArgument } from 'n8n-workflow'; +import { z } from 'zod'; + +export type CreateNodeAsToolOptions = { + node: INode; + nodeType: INodeType; + handleToolInvocation: (toolArgs: IDataObject) => Promise; +}; + +/** + * Retrieves and validates the Zod schema for the tool. + * + * This method: + * 1. Collects all $fromAI arguments from node parameters + * 2. Validates parameter keys against naming rules + * 3. Checks for duplicate keys and ensures consistency + * 4. Generates a Zod schema from the validated arguments + * + * @throws {NodeOperationError} When parameter keys are invalid or when duplicate keys have inconsistent definitions + * @returns {z.ZodObject} A Zod schema object representing the structure and validation rules for the node parameters + */ +function getSchema(node: INode) { + const collectedArguments: FromAIArgument[] = []; + try { + traverseNodeParameters(node.parameters, collectedArguments); + } catch (error) { + throw new NodeOperationError(node, error as Error); + } + + // Validate each collected argument + const nameValidationRegex = /^[a-zA-Z0-9_-]{1,64}$/; + const keyMap = new Map(); + for (const argument of collectedArguments) { + if (argument.key.length === 0 || !nameValidationRegex.test(argument.key)) { + const isEmptyError = 'You must specify a key when using $fromAI()'; + const isInvalidError = `Parameter key \`${argument.key}\` is invalid`; + const error = new Error(argument.key.length === 0 ? isEmptyError : isInvalidError); + throw new NodeOperationError(node, error, { + description: + 'Invalid parameter key, must be between 1 and 64 characters long and only contain letters, numbers, underscores, and hyphens', + }); + } + + if (keyMap.has(argument.key)) { + // If the key already exists in the Map + const existingArg = keyMap.get(argument.key)!; + + // Check if the existing argument has the same description and type + if (existingArg.description !== argument.description || existingArg.type !== argument.type) { + // If not, throw an error for inconsistent duplicate keys + throw new NodeOperationError( + node, + `Duplicate key '${argument.key}' found with different description or type`, + { + description: + 'Ensure all $fromAI() calls with the same key have consistent descriptions and types', + }, + ); + } + // If the duplicate key has consistent description and type, it's allowed (no action needed) + } else { + // If the key doesn't exist in the Map, add it + keyMap.set(argument.key, argument); + } + } + + // Remove duplicate keys, latest occurrence takes precedence + const uniqueArgsMap = collectedArguments.reduce((map, arg) => { + map.set(arg.key, arg); + return map; + }, new Map()); + + const uniqueArguments = Array.from(uniqueArgsMap.values()); + + // Generate Zod schema from unique arguments + const schemaObj = uniqueArguments.reduce((acc: Record, placeholder) => { + acc[placeholder.key] = generateZodSchema(placeholder); + return acc; + }, {}); + + return z.object(schemaObj).required(); +} + +/** + * Generates a description for a node based on the provided parameters. + * @param node The node type. + * @param nodeParameters The parameters of the node. + * @returns A string description for the node. + */ +function makeDescription(node: INode, nodeType: INodeType): string { + const manualDescription = node.parameters.toolDescription as string; + + if (node.parameters.descriptionType === 'auto') { + const resource = node.parameters.resource as string; + const operation = node.parameters.operation as string; + let description = nodeType.description.description; + if (resource) { + description += `\n Resource: ${resource}`; + } + if (operation) { + description += `\n Operation: ${operation}`; + } + return description.trim(); + } + if (node.parameters.descriptionType === 'manual') { + return manualDescription ?? nodeType.description.description; + } + + return nodeType.description.description; +} + +/** + * Creates a DynamicStructuredTool from a node. + * @returns A DynamicStructuredTool instance. + */ +function createTool(options: CreateNodeAsToolOptions) { + const { node, nodeType, handleToolInvocation } = options; + const schema = getSchema(node); + const description = makeDescription(node, nodeType); + const nodeName = node.name.replace(/ /g, '_'); + const name = nodeName || nodeType.description.name; + + return new DynamicStructuredTool({ + name, + description, + schema, + func: async (toolArgs: z.infer) => await handleToolInvocation(toolArgs), + }); +} + +/** + * Converts node into LangChain tool by analyzing node parameters, + * identifying placeholders using the $fromAI function, and generating a Zod schema. It then creates + * a DynamicStructuredTool that can be used in LangChain workflows. + */ +export function createNodeAsTool(options: CreateNodeAsToolOptions) { + return { response: createTool(options) }; +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/deduplication-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/deduplication-helper-functions.ts new file mode 100644 index 0000000000..f1a65724a5 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/deduplication-helper-functions.ts @@ -0,0 +1,128 @@ +import type { + IDataObject, + INode, + Workflow, + DeduplicationHelperFunctions, + IDeduplicationOutput, + IDeduplicationOutputItems, + ICheckProcessedOptions, + DeduplicationScope, + DeduplicationItemTypes, + ICheckProcessedContextData, +} from 'n8n-workflow'; + +import { DataDeduplicationService } from '@/data-deduplication-service'; + +async function checkProcessedAndRecord( + items: DeduplicationItemTypes[], + scope: DeduplicationScope, + contextData: ICheckProcessedContextData, + options: ICheckProcessedOptions, +): Promise { + return await DataDeduplicationService.getInstance().checkProcessedAndRecord( + items, + scope, + contextData, + options, + ); +} + +async function checkProcessedItemsAndRecord( + key: string, + items: IDataObject[], + scope: DeduplicationScope, + contextData: ICheckProcessedContextData, + options: ICheckProcessedOptions, +): Promise { + return await DataDeduplicationService.getInstance().checkProcessedItemsAndRecord( + key, + items, + scope, + contextData, + options, + ); +} + +async function removeProcessed( + items: DeduplicationItemTypes[], + scope: DeduplicationScope, + contextData: ICheckProcessedContextData, + options: ICheckProcessedOptions, +): Promise { + return await DataDeduplicationService.getInstance().removeProcessed( + items, + scope, + contextData, + options, + ); +} + +async function clearAllProcessedItems( + scope: DeduplicationScope, + contextData: ICheckProcessedContextData, + options: ICheckProcessedOptions, +): Promise { + return await DataDeduplicationService.getInstance().clearAllProcessedItems( + scope, + contextData, + options, + ); +} + +async function getProcessedDataCount( + scope: DeduplicationScope, + contextData: ICheckProcessedContextData, + options: ICheckProcessedOptions, +): Promise { + return await DataDeduplicationService.getInstance().getProcessedDataCount( + scope, + contextData, + options, + ); +} + +export const getDeduplicationHelperFunctions = ( + workflow: Workflow, + node: INode, +): DeduplicationHelperFunctions => ({ + async checkProcessedAndRecord( + items: DeduplicationItemTypes[], + scope: DeduplicationScope, + options: ICheckProcessedOptions, + ): Promise { + return await checkProcessedAndRecord(items, scope, { node, workflow }, options); + }, + async checkProcessedItemsAndRecord( + propertyName: string, + items: IDataObject[], + scope: DeduplicationScope, + options: ICheckProcessedOptions, + ): Promise { + return await checkProcessedItemsAndRecord( + propertyName, + items, + scope, + { node, workflow }, + options, + ); + }, + async removeProcessed( + items: DeduplicationItemTypes[], + scope: DeduplicationScope, + options: ICheckProcessedOptions, + ): Promise { + return await removeProcessed(items, scope, { node, workflow }, options); + }, + async clearAllProcessedItems( + scope: DeduplicationScope, + options: ICheckProcessedOptions, + ): Promise { + return await clearAllProcessedItems(scope, { node, workflow }, options); + }, + async getProcessedDataCount( + scope: DeduplicationScope, + options: ICheckProcessedOptions, + ): Promise { + return await getProcessedDataCount(scope, { node, workflow }, options); + }, +}); diff --git a/packages/core/src/node-execution-context/utils/ensureType.ts b/packages/core/src/execution-engine/node-execution-context/utils/ensure-type.ts similarity index 100% rename from packages/core/src/node-execution-context/utils/ensureType.ts rename to packages/core/src/execution-engine/node-execution-context/utils/ensure-type.ts diff --git a/packages/core/src/ExecutionMetadata.ts b/packages/core/src/execution-engine/node-execution-context/utils/execution-metadata.ts similarity index 92% rename from packages/core/src/ExecutionMetadata.ts rename to packages/core/src/execution-engine/node-execution-context/utils/execution-metadata.ts index 8466933e05..29957b983a 100644 --- a/packages/core/src/ExecutionMetadata.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/execution-metadata.ts @@ -1,7 +1,7 @@ import type { IRunExecutionData } from 'n8n-workflow'; import { LoggerProxy as Logger } from 'n8n-workflow'; -import { InvalidExecutionMetadataError } from './errors/invalid-execution-metadata.error'; +import { InvalidExecutionMetadataError } from '@/errors/invalid-execution-metadata.error'; export const KV_LIMIT = 10; @@ -38,9 +38,9 @@ export function setWorkflowExecutionMetadata( Logger.error('Custom data key over 50 characters long. Truncating to 50 characters.'); } if (val.length > 255) { - Logger.error('Custom data value over 255 characters long. Truncating to 255 characters.'); + Logger.error('Custom data value over 512 characters long. Truncating to 512 characters.'); } - executionData.resultData.metadata[key.slice(0, 50)] = val.slice(0, 255); + executionData.resultData.metadata[key.slice(0, 50)] = val.slice(0, 512); } export function setAllWorkflowExecutionMetadata( diff --git a/packages/core/src/ExtractValue.ts b/packages/core/src/execution-engine/node-execution-context/utils/extract-value.ts similarity index 100% rename from packages/core/src/ExtractValue.ts rename to packages/core/src/execution-engine/node-execution-context/utils/extract-value.ts diff --git a/packages/core/src/execution-engine/node-execution-context/utils/file-system-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/file-system-helper-functions.ts new file mode 100644 index 0000000000..23153bb94d --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/file-system-helper-functions.ts @@ -0,0 +1,121 @@ +import { Container } from '@n8n/di'; +import type { FileSystemHelperFunctions, INode } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; +import { createReadStream } from 'node:fs'; +import { access as fsAccess, writeFile as fsWriteFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +import { + BINARY_DATA_STORAGE_PATH, + BLOCK_FILE_ACCESS_TO_N8N_FILES, + CONFIG_FILES, + CUSTOM_EXTENSION_ENV, + RESTRICT_FILE_ACCESS_TO, + UM_EMAIL_TEMPLATES_INVITE, + UM_EMAIL_TEMPLATES_PWRESET, +} from '@/constants'; +import { InstanceSettings } from '@/instance-settings'; + +const getAllowedPaths = () => { + const restrictFileAccessTo = process.env[RESTRICT_FILE_ACCESS_TO]; + if (!restrictFileAccessTo) { + return []; + } + const allowedPaths = restrictFileAccessTo + .split(';') + .map((path) => path.trim()) + .filter((path) => path); + return allowedPaths; +}; + +export function isFilePathBlocked(filePath: string): boolean { + const allowedPaths = getAllowedPaths(); + const resolvedFilePath = resolve(filePath); + const blockFileAccessToN8nFiles = process.env[BLOCK_FILE_ACCESS_TO_N8N_FILES] !== 'false'; + + //if allowed paths are defined, allow access only to those paths + if (allowedPaths.length) { + for (const path of allowedPaths) { + if (resolvedFilePath.startsWith(path)) { + return false; + } + } + + return true; + } + + //restrict access to .n8n folder, ~/.cache/n8n/public, and other .env config related paths + if (blockFileAccessToN8nFiles) { + const { n8nFolder, staticCacheDir } = Container.get(InstanceSettings); + const restrictedPaths = [n8nFolder, staticCacheDir]; + + if (process.env[CONFIG_FILES]) { + restrictedPaths.push(...process.env[CONFIG_FILES].split(',')); + } + + if (process.env[CUSTOM_EXTENSION_ENV]) { + const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV].split(';'); + restrictedPaths.push(...customExtensionFolders); + } + + if (process.env[BINARY_DATA_STORAGE_PATH]) { + restrictedPaths.push(process.env[BINARY_DATA_STORAGE_PATH]); + } + + if (process.env[UM_EMAIL_TEMPLATES_INVITE]) { + restrictedPaths.push(process.env[UM_EMAIL_TEMPLATES_INVITE]); + } + + if (process.env[UM_EMAIL_TEMPLATES_PWRESET]) { + restrictedPaths.push(process.env[UM_EMAIL_TEMPLATES_PWRESET]); + } + + //check if the file path is restricted + for (const path of restrictedPaths) { + if (resolvedFilePath.startsWith(path)) { + return true; + } + } + } + + //path is not restricted + return false; +} + +export const getFileSystemHelperFunctions = (node: INode): FileSystemHelperFunctions => ({ + async createReadStream(filePath) { + try { + await fsAccess(filePath); + } catch (error) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + throw error.code === 'ENOENT' + ? // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + new NodeOperationError(node, error, { + message: `The file "${String(filePath)}" could not be accessed.`, + level: 'warning', + }) + : error; + } + if (isFilePathBlocked(filePath as string)) { + const allowedPaths = getAllowedPaths(); + const message = allowedPaths.length ? ` Allowed paths: ${allowedPaths.join(', ')}` : ''; + throw new NodeOperationError(node, `Access to the file is not allowed.${message}`, { + level: 'warning', + }); + } + return createReadStream(filePath); + }, + + getStoragePath() { + return join(Container.get(InstanceSettings).n8nFolder, `storage/${node.type}`); + }, + + async writeContentToFile(filePath, content, flag) { + if (isFilePathBlocked(filePath as string)) { + throw new NodeOperationError(node, `The file "${String(filePath)}" is not writable.`, { + level: 'warning', + }); + } + return await fsWriteFile(filePath, content, { encoding: 'binary', flag }); + }, +}); diff --git a/packages/core/src/node-execution-context/utils/getAdditionalKeys.ts b/packages/core/src/execution-engine/node-execution-context/utils/get-additional-keys.ts similarity index 93% rename from packages/core/src/node-execution-context/utils/getAdditionalKeys.ts rename to packages/core/src/execution-engine/node-execution-context/utils/get-additional-keys.ts index 28bf3b89f6..7e18d0f2d1 100644 --- a/packages/core/src/node-execution-context/utils/getAdditionalKeys.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/get-additional-keys.ts @@ -6,14 +6,15 @@ import type { } from 'n8n-workflow'; import { LoggerProxy } from 'n8n-workflow'; -import { PLACEHOLDER_EMPTY_EXECUTION_ID } from '@/Constants'; +import { PLACEHOLDER_EMPTY_EXECUTION_ID } from '@/constants'; + import { setWorkflowExecutionMetadata, setAllWorkflowExecutionMetadata, getWorkflowExecutionMetadata, getAllWorkflowExecutionMetadata, -} from '@/ExecutionMetadata'; -import { getSecretsProxy } from '@/Secrets'; +} from './execution-metadata'; +import { getSecretsProxy } from './get-secrets-proxy'; /** Returns the additional keys for Expressions and Function-Nodes */ export function getAdditionalKeys( diff --git a/packages/core/src/node-execution-context/utils/getInputConnectionData.ts b/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts similarity index 96% rename from packages/core/src/node-execution-context/utils/getInputConnectionData.ts rename to packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts index 231f672d65..9f76d0e106 100644 --- a/packages/core/src/node-execution-context/utils/getInputConnectionData.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts @@ -19,10 +19,10 @@ import { ApplicationError, } from 'n8n-workflow'; -import { createNodeAsTool } from '@/CreateNodeAsTool'; +import { createNodeAsTool } from './create-node-as-tool'; // eslint-disable-next-line import/no-cycle -import { SupplyDataContext } from '@/node-execution-context'; -import type { ExecuteContext, WebhookContext } from '@/node-execution-context'; +import { SupplyDataContext } from '../../node-execution-context'; +import type { ExecuteContext, WebhookContext } from '../../node-execution-context'; export async function getInputConnectionData( this: ExecuteContext | WebhookContext | SupplyDataContext, diff --git a/packages/core/src/Secrets.ts b/packages/core/src/execution-engine/node-execution-context/utils/get-secrets-proxy.ts similarity index 100% rename from packages/core/src/Secrets.ts rename to packages/core/src/execution-engine/node-execution-context/utils/get-secrets-proxy.ts diff --git a/packages/core/src/execution-engine/node-execution-context/utils/parse-incoming-message.ts b/packages/core/src/execution-engine/node-execution-context/utils/parse-incoming-message.ts new file mode 100644 index 0000000000..148e59be86 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/parse-incoming-message.ts @@ -0,0 +1,95 @@ +import type { IncomingMessage } from 'http'; + +function parseHeaderParameters(parameters: string[]): Record { + return parameters.reduce( + (acc, param) => { + const [key, value] = param.split('='); + let decodedValue = decodeURIComponent(value).trim(); + if (decodedValue.startsWith('"') && decodedValue.endsWith('"')) { + decodedValue = decodedValue.slice(1, -1); + } + acc[key.toLowerCase().trim()] = decodedValue; + return acc; + }, + {} as Record, + ); +} + +interface IContentType { + type: string; + parameters: { + charset: string; + [key: string]: string; + }; +} + +/** + * Parses the Content-Type header string into a structured object + * @returns {IContentType | null} Parsed content type details or null if no content type is detected + */ +export const parseContentType = (contentType?: string): IContentType | null => { + if (!contentType) { + return null; + } + + const [type, ...parameters] = contentType.split(';'); + + return { + type: type.toLowerCase(), + parameters: { charset: 'utf-8', ...parseHeaderParameters(parameters) }, + }; +}; + +interface IContentDisposition { + type: string; + filename?: string; +} + +/** + * Parses the Content-Disposition header string into a structured object + * @returns {IContentDisposition | null} Parsed content disposition details or null if no content disposition is detected + */ +export const parseContentDisposition = ( + contentDisposition?: string, +): IContentDisposition | null => { + if (!contentDisposition) { + return null; + } + + // This is invalid syntax, but common + // Example 'filename="example.png"' (instead of 'attachment; filename="example.png"') + if (!contentDisposition.startsWith('attachment') && !contentDisposition.startsWith('inline')) { + contentDisposition = `attachment; ${contentDisposition}`; + } + + const [type, ...parameters] = contentDisposition.split(';'); + + const parsedParameters = parseHeaderParameters(parameters); + + let { filename } = parsedParameters; + const wildcard = parsedParameters['filename*']; + if (wildcard) { + // https://datatracker.ietf.org/doc/html/rfc5987 + const [_encoding, _locale, content] = wildcard?.split("'") ?? []; + filename = content; + } + + return { type, filename }; +}; + +/** + * Augments an IncomingMessage with parsed content type and disposition information + */ +export function parseIncomingMessage(message: IncomingMessage) { + const contentType = parseContentType(message.headers['content-type']); + if (contentType) { + const { type, parameters } = contentType; + message.contentType = type; + message.encoding = parameters.charset.toLowerCase() as BufferEncoding; + } + + const contentDisposition = parseContentDisposition(message.headers['content-disposition']); + if (contentDisposition) { + message.contentDisposition = contentDisposition; + } +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts b/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts new file mode 100644 index 0000000000..fb80eff52d --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts @@ -0,0 +1,468 @@ +import { Container } from '@n8n/di'; +import type { AxiosHeaders, AxiosRequestConfig } from 'axios'; +import crypto from 'crypto'; +import FormData from 'form-data'; +import { Agent, type AgentOptions } from 'https'; +import type { GenericValue, IRequestOptions } from 'n8n-workflow'; +import { stringify } from 'qs'; +import { URL } from 'url'; + +import { Logger } from '@/logging/logger'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const pushFormDataValue = (form: FormData, key: string, value: any) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + if (value?.hasOwnProperty('value') && value.hasOwnProperty('options')) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument + form.append(key, value.value, value.options); + } else { + form.append(key, value); + } +}; + +const createFormDataObject = (data: Record) => { + const formData = new FormData(); + const keys = Object.keys(data); + keys.forEach((key) => { + const formField = data[key]; + + if (formField instanceof Array) { + formField.forEach((item) => { + pushFormDataValue(formData, key, item); + }); + } else { + pushFormDataValue(formData, key, formField); + } + }); + return formData; +}; + +function searchForHeader(config: AxiosRequestConfig, headerName: string) { + if (config.headers === undefined) { + return undefined; + } + + const headerNames = Object.keys(config.headers); + headerName = headerName.toLowerCase(); + return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName); +} + +async function generateContentLengthHeader(config: AxiosRequestConfig) { + if (!(config.data instanceof FormData)) { + return; + } + try { + const length = await new Promise((res, rej) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + config.data.getLength((error: Error | null, dataLength: number) => { + if (error) rej(error); + else res(dataLength); + }); + }); + config.headers = { + ...config.headers, + 'content-length': length, + }; + } catch (error) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + Container.get(Logger).error('Unable to calculate form data length', { error }); + } +} + +const getHostFromRequestObject = ( + requestObject: Partial<{ + url: string; + uri: string; + baseURL: string; + }>, +): string | null => { + try { + const url = (requestObject.url ?? requestObject.uri) as string; + return new URL(url, requestObject.baseURL).hostname; + } catch (error) { + return null; + } +}; + +const getBeforeRedirectFn = + (agentOptions: AgentOptions, axiosConfig: AxiosRequestConfig) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (redirectedRequest: Record) => { + const redirectAgent = new Agent({ + ...agentOptions, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + servername: redirectedRequest.hostname, + }); + redirectedRequest.agent = redirectAgent; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + redirectedRequest.agents.https = redirectAgent; + + if (axiosConfig.headers?.Authorization) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + redirectedRequest.headers.Authorization = axiosConfig.headers.Authorization; + } + if (axiosConfig.auth) { + redirectedRequest.auth = `${axiosConfig.auth.username}:${axiosConfig.auth.password}`; + } + }; + +/** + * This function is a temporary implementation that translates all http requests + * done via the request library to axios directly. + * We are not using n8n's interface as it would an unnecessary step, + * considering the `request` helper has been be deprecated and should be removed. + * @deprecated This is only used by legacy request helpers, that are also deprecated + */ +// eslint-disable-next-line complexity +export async function parseRequestObject(requestObject: IRequestOptions) { + const axiosConfig: AxiosRequestConfig = {}; + + if (requestObject.headers !== undefined) { + axiosConfig.headers = requestObject.headers as AxiosHeaders; + } + + // Let's start parsing the hardest part, which is the request body. + // The process here is as following? + // - Check if we have a `content-type` header. If this was set, + // we will follow + // - Check if the `form` property was set. If yes, then it's x-www-form-urlencoded + // - Check if the `formData` property exists. If yes, then it's multipart/form-data + // - Lastly, we should have a regular `body` that is probably a JSON. + + const contentTypeHeaderKeyName = + axiosConfig.headers && + Object.keys(axiosConfig.headers).find( + (headerName) => headerName.toLowerCase() === 'content-type', + ); + const contentType = + contentTypeHeaderKeyName && + (axiosConfig.headers?.[contentTypeHeaderKeyName] as string | undefined); + if (contentType === 'application/x-www-form-urlencoded' && requestObject.formData === undefined) { + // there are nodes incorrectly created, informing the content type header + // and also using formData. Request lib takes precedence for the formData. + // We will do the same. + // Merge body and form properties. + if (typeof requestObject.body === 'string') { + axiosConfig.data = requestObject.body; + } else { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + const allData = Object.assign(requestObject.body || {}, requestObject.form || {}) as Record< + string, + string + >; + if (requestObject.useQuerystring === true) { + axiosConfig.data = stringify(allData, { arrayFormat: 'repeat' }); + } else { + axiosConfig.data = stringify(allData); + } + } + } else if (contentType?.includes('multipart/form-data')) { + if (requestObject.formData !== undefined && requestObject.formData instanceof FormData) { + axiosConfig.data = requestObject.formData; + } else { + const allData: Partial = { + ...(requestObject.body as object | undefined), + ...(requestObject.formData as object | undefined), + }; + + axiosConfig.data = createFormDataObject(allData); + } + // replace the existing header with a new one that + // contains the boundary property. + delete axiosConfig.headers?.[contentTypeHeaderKeyName!]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + const headers = axiosConfig.data.getHeaders(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); + await generateContentLengthHeader(axiosConfig); + } else { + // When using the `form` property it means the content should be x-www-form-urlencoded. + if (requestObject.form !== undefined && requestObject.body === undefined) { + // If we have only form + axiosConfig.data = + typeof requestObject.form === 'string' + ? stringify(requestObject.form, { format: 'RFC3986' }) + : stringify(requestObject.form).toString(); + if (axiosConfig.headers !== undefined) { + const headerName = searchForHeader(axiosConfig, 'content-type'); + if (headerName) { + delete axiosConfig.headers[headerName]; + } + axiosConfig.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } else { + axiosConfig.headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + }; + } + } else if (requestObject.formData !== undefined) { + // remove any "content-type" that might exist. + if (axiosConfig.headers !== undefined) { + const headers = Object.keys(axiosConfig.headers); + headers.forEach((header) => { + if (header.toLowerCase() === 'content-type') { + delete axiosConfig.headers?.[header]; + } + }); + } + + if (requestObject.formData instanceof FormData) { + axiosConfig.data = requestObject.formData; + } else { + axiosConfig.data = createFormDataObject(requestObject.formData as Record); + } + // Mix in headers as FormData creates the boundary. + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + const headers = axiosConfig.data.getHeaders(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); + await generateContentLengthHeader(axiosConfig); + } else if (requestObject.body !== undefined) { + // If we have body and possibly form + if (requestObject.form !== undefined && requestObject.body) { + // merge both objects when exist. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + requestObject.body = Object.assign(requestObject.body, requestObject.form); + } + axiosConfig.data = requestObject.body as FormData | GenericValue | GenericValue[]; + } + } + + if (requestObject.uri !== undefined) { + axiosConfig.url = requestObject.uri?.toString(); + } + + if (requestObject.url !== undefined) { + axiosConfig.url = requestObject.url?.toString(); + } + + if (requestObject.baseURL !== undefined) { + axiosConfig.baseURL = requestObject.baseURL?.toString(); + } + + if (requestObject.method !== undefined) { + axiosConfig.method = requestObject.method; + } + + if (requestObject.qs !== undefined && Object.keys(requestObject.qs as object).length > 0) { + axiosConfig.params = requestObject.qs; + } + + function hasArrayFormatOptions( + arg: IRequestOptions, + ): arg is Required> { + if ( + typeof arg.qsStringifyOptions === 'object' && + arg.qsStringifyOptions !== null && + !Array.isArray(arg.qsStringifyOptions) && + 'arrayFormat' in arg.qsStringifyOptions + ) { + return true; + } + + return false; + } + + if ( + requestObject.useQuerystring === true || + (hasArrayFormatOptions(requestObject) && + requestObject.qsStringifyOptions.arrayFormat === 'repeat') + ) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'repeat' }); + }; + } else if (requestObject.useQuerystring === false) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'indices' }); + }; + } + + if ( + hasArrayFormatOptions(requestObject) && + requestObject.qsStringifyOptions.arrayFormat === 'brackets' + ) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'brackets' }); + }; + } + + if (requestObject.auth !== undefined) { + // Check support for sendImmediately + if (requestObject.auth.bearer !== undefined) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + Authorization: `Bearer ${requestObject.auth.bearer}`, + }); + } else { + const authObj = requestObject.auth; + // Request accepts both user/username and pass/password + axiosConfig.auth = { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + username: (authObj.user || authObj.username) as string, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + password: (authObj.password || authObj.pass) as string, + }; + } + } + + // Only set header if we have a body, otherwise it may fail + if (requestObject.json === true) { + // Add application/json headers - do not set charset as it breaks a lot of stuff + // only add if no other accept headers was sent. + const acceptHeaderExists = + axiosConfig.headers === undefined + ? false + : Object.keys(axiosConfig.headers) + .map((headerKey) => headerKey.toLowerCase()) + .includes('accept'); + if (!acceptHeaderExists) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + Accept: 'application/json', + }); + } + } + if (requestObject.json === false || requestObject.json === undefined) { + // Prevent json parsing + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + axiosConfig.transformResponse = (res) => res; + } + + // Axios will follow redirects by default, so we simply tell it otherwise if needed. + const { method } = requestObject; + if ( + (requestObject.followRedirect !== false && + (!method || method === 'GET' || method === 'HEAD')) || + requestObject.followAllRedirects + ) { + axiosConfig.maxRedirects = requestObject.maxRedirects; + } else { + axiosConfig.maxRedirects = 0; + } + + const host = getHostFromRequestObject(requestObject); + const agentOptions: AgentOptions = { ...requestObject.agentOptions }; + if (host) { + agentOptions.servername = host; + } + if (requestObject.rejectUnauthorized === false) { + agentOptions.rejectUnauthorized = false; + agentOptions.secureOptions = crypto.constants.SSL_OP_LEGACY_SERVER_CONNECT; + } + + axiosConfig.httpsAgent = new Agent(agentOptions); + + axiosConfig.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosConfig); + + if (requestObject.timeout !== undefined) { + axiosConfig.timeout = requestObject.timeout; + } + + if (requestObject.proxy !== undefined) { + // try our best to parse the url provided. + if (typeof requestObject.proxy === 'string') { + try { + const url = new URL(requestObject.proxy); + // eslint-disable-next-line @typescript-eslint/no-shadow + const host = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; + axiosConfig.proxy = { + host, + port: parseInt(url.port, 10), + protocol: url.protocol, + }; + if (!url.port) { + // Sets port to a default if not informed + if (url.protocol === 'http') { + axiosConfig.proxy.port = 80; + } else if (url.protocol === 'https') { + axiosConfig.proxy.port = 443; + } + } + if (url.username || url.password) { + axiosConfig.proxy.auth = { + username: url.username, + password: url.password, + }; + } + } catch (error) { + // Not a valid URL. We will try to simply parse stuff + // such as user:pass@host:port without protocol (we'll assume http) + if (requestObject.proxy.includes('@')) { + const [userpass, hostport] = requestObject.proxy.split('@'); + const [username, password] = userpass.split(':'); + const [hostname, port] = hostport.split(':'); + // eslint-disable-next-line @typescript-eslint/no-shadow + const host = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname; + axiosConfig.proxy = { + host, + port: parseInt(port, 10), + protocol: 'http', + auth: { + username, + password, + }, + }; + } else if (requestObject.proxy.includes(':')) { + const [hostname, port] = requestObject.proxy.split(':'); + axiosConfig.proxy = { + host: hostname, + port: parseInt(port, 10), + protocol: 'http', + }; + } else { + axiosConfig.proxy = { + host: requestObject.proxy, + port: 80, + protocol: 'http', + }; + } + } + } else { + axiosConfig.proxy = requestObject.proxy; + } + } + + if (requestObject.useStream) { + axiosConfig.responseType = 'stream'; + } else if (requestObject.encoding === null) { + // When downloading files, return an arrayBuffer. + axiosConfig.responseType = 'arraybuffer'; + } + + // If we don't set an accept header + // Axios forces "application/json, text/plan, */*" + // Which causes some nodes like NextCloud to break + // as the service returns XML unless requested otherwise. + const allHeaders = axiosConfig.headers ? Object.keys(axiosConfig.headers) : []; + if (!allHeaders.some((headerKey) => headerKey.toLowerCase() === 'accept')) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { accept: '*/*' }); + } + if ( + requestObject.json !== false && + axiosConfig.data !== undefined && + axiosConfig.data !== '' && + !(axiosConfig.data instanceof Buffer) && + !allHeaders.some((headerKey) => headerKey.toLowerCase() === 'content-type') + ) { + // Use default header for application/json + // If we don't specify this here, axios will add + // application/json; charset=utf-8 + // and this breaks a lot of stuff + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + 'content-type': 'application/json', + }); + } + + if (requestObject.simple === false) { + axiosConfig.validateStatus = () => true; + } + + /** + * Missing properties: + * encoding (need testing) + * gzip (ignored - default already works) + * resolveWithFullResponse (implemented elsewhere) + */ + return axiosConfig; +} diff --git a/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts b/packages/core/src/execution-engine/node-execution-context/utils/validate-value-against-schema.ts similarity index 90% rename from packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts rename to packages/core/src/execution-engine/node-execution-context/utils/validate-value-against-schema.ts index d058c50a52..73b5423f49 100644 --- a/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/validate-value-against-schema.ts @@ -6,6 +6,7 @@ import type { INodePropertyCollection, INodePropertyOptions, INodeType, + ResourceMapperTypeOptions, } from 'n8n-workflow'; import { ExpressionError, @@ -14,15 +15,17 @@ import { validateFieldType, } from 'n8n-workflow'; -import type { ExtendedValidationResult } from '@/Interfaces'; +import type { ExtendedValidationResult } from '@/interfaces'; const validateResourceMapperValue = ( parameterName: string, paramValues: { [key: string]: unknown }, node: INode, - skipRequiredCheck = false, + resourceMapperTypeOptions?: ResourceMapperTypeOptions, ): ExtendedValidationResult => { const result: ExtendedValidationResult = { valid: true, newValue: paramValues }; + const skipRequiredCheck = resourceMapperTypeOptions?.mode !== 'add'; + const enableTypeValidationOptions = Boolean(resourceMapperTypeOptions?.showTypeConversionOptions); const paramNameParts = parameterName.split('.'); if (paramNameParts.length !== 2) { return result; @@ -44,7 +47,7 @@ const validateResourceMapperValue = ( !skipRequiredCheck && schemaEntry?.required === true && schemaEntry.type !== 'boolean' && - !resolvedValue + (resolvedValue === undefined || resolvedValue === null) ) { return { valid: false, @@ -56,8 +59,8 @@ const validateResourceMapperValue = ( if (schemaEntry?.type) { const validationResult = validateFieldType(key, resolvedValue, schemaEntry.type, { valueOptions: schemaEntry.options, - strict: !resourceMapperField.attemptToConvertTypes, - parseStrings: !!resourceMapperField.convertFieldsToString, + strict: enableTypeValidationOptions && !resourceMapperField.attemptToConvertTypes, + parseStrings: enableTypeValidationOptions && resourceMapperField.convertFieldsToString, }); if (!validationResult.valid) { @@ -185,7 +188,7 @@ export const validateValueAgainstSchema = ( parameterName, parameterValue as { [key: string]: unknown }, node, - propertyDescription.typeOptions?.resourceMapper?.mode !== 'add', + propertyDescription.typeOptions?.resourceMapper, ); } else if (['fixedCollection', 'collection'].includes(propertyDescription.type)) { validationResult = validateCollection( diff --git a/packages/core/src/node-execution-context/webhook-context.ts b/packages/core/src/execution-engine/node-execution-context/webhook-context.ts similarity index 95% rename from packages/core/src/node-execution-context/webhook-context.ts rename to packages/core/src/execution-engine/node-execution-context/webhook-context.ts index 9d131a4103..dba5c8124c 100644 --- a/packages/core/src/node-execution-context/webhook-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/webhook-context.ts @@ -20,15 +20,14 @@ import { ApplicationError, createDeferredPromise } from 'n8n-workflow'; // eslint-disable-next-line import/no-cycle import { - copyBinaryFile, - getBinaryHelperFunctions, getNodeWebhookUrl, getRequestHelperFunctions, returnJsonArray, -} from '@/NodeExecuteFunctions'; +} from '@/node-execute-functions'; import { NodeExecutionContext } from './node-execution-context'; -import { getInputConnectionData } from './utils/getInputConnectionData'; +import { copyBinaryFile, getBinaryHelperFunctions } from './utils/binary-helper-functions'; +import { getInputConnectionData } from './utils/get-input-connection-data'; export class WebhookContext extends NodeExecutionContext implements IWebhookFunctions { readonly helpers: IWebhookFunctions['helpers']; diff --git a/packages/core/src/node-execution-context/workflow-node-context.ts b/packages/core/src/execution-engine/node-execution-context/workflow-node-context.ts similarity index 100% rename from packages/core/src/node-execution-context/workflow-node-context.ts rename to packages/core/src/execution-engine/node-execution-context/workflow-node-context.ts diff --git a/packages/core/src/execution-engine/partial-execution-utils/__tests__/clean-run-data.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/clean-run-data.test.ts new file mode 100644 index 0000000000..1e46e40070 --- /dev/null +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/clean-run-data.test.ts @@ -0,0 +1,234 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to +// XX denotes that the node is disabled +// PD denotes that the node has pinned data + +import { NodeConnectionType, type IRunData } from 'n8n-workflow'; + +import { createNodeData, toITaskData } from './helpers'; +import { cleanRunData } from '../clean-run-data'; +import { DirectedGraph } from '../directed-graph'; + +describe('cleanRunData', () => { + // ┌─────┐ ┌─────┐ ┌─────┐ + // │node1├───►│node2├──►│node3│ + // └─────┘ └─────┘ └─────┘ + test('deletes all run data of all children and the node being passed in', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 2 } }])], + [node3.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([node1])); + + // ASSERT + expect(newRunData).toEqual({}); + }); + + // ┌─────┐ ┌─────┐ ┌─────┐ + // │node1├───►│node2├──►│node3│ + // └─────┘ └─────┘ └─────┘ + test('retains the run data of parent nodes of the node being passed in', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 2 } }])], + [node3.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([node2])); + + // ASSERT + expect(newRunData).toEqual({ [node1.name]: runData[node1.name] }); + }); + + // ┌─────┐ ┌─────┐ ┌─────┐ + // ┌─►│node1├───►│node2├──►│node3├─┐ + // │ └─────┘ └─────┘ └─────┘ │ + // │ │ + // └───────────────────────────────┘ + test('terminates when finding a cycle', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections( + { from: node1, to: node2 }, + { from: node2, to: node3 }, + { from: node3, to: node1 }, + ); + + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 2 } }])], + [node3.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([node2])); + + // ASSERT + // TODO: Find out if this is a desirable result in milestone 2 + expect(newRunData).toEqual({}); + }); + + // ┌─────┐ ┌─────┐ + // │node1├───►│node2│ + // └─────┘ └─────┘ + test('removes run data of nodes that are not in the subgraph', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const graph = new DirectedGraph() + .addNodes(node1, node2) + .addConnections({ from: node1, to: node2 }); + // not part of the graph + const node3 = createNodeData({ name: 'Node3' }); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 2 } }])], + [node3.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([node2])); + + // ASSERT + expect(newRunData).toEqual({ + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }); + }); + + // ► + // ┌─────┐ ┌────────┐ + // │node1├─────►rootNode│ + // └─────┘ └───▲────┘ + // │ + // ┌───┴───┐ + // │subNode│ + // └───────┘ + test('removes run data of sub nodes when the start node is a root node', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const rootNode = createNodeData({ name: 'Root Node' }); + const subNode = createNodeData({ name: 'Sub Node' }); + const graph = new DirectedGraph() + .addNodes(node1, rootNode, subNode) + .addConnections( + { from: node1, to: rootNode }, + { from: subNode, to: rootNode, type: NodeConnectionType.AiLanguageModel }, + ); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [rootNode.name]: [toITaskData([{ data: { value: 2 } }])], + [subNode.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([rootNode])); + + // ASSERT + expect(newRunData).toEqual({ + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }); + }); + + // ► + // ┌─────┐ ┌─────┐ ┌────────┐ + // │node1├───►node2├────►rootNode│ + // └─────┘ └─────┘ └───▲────┘ + // │ + // ┌───┴───┐ + // │subNode│ + // └───────┘ + test('removes run data of sub nodes for root nodes downstream of the start node', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const rootNode = createNodeData({ name: 'Root Node' }); + const subNode = createNodeData({ name: 'Sub Node' }); + const graph = new DirectedGraph() + .addNodes(node1, node2, rootNode, subNode) + .addConnections( + { from: node1, to: node2 }, + { from: node2, to: rootNode }, + { from: subNode, to: rootNode, type: NodeConnectionType.AiLanguageModel }, + ); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 1 } }])], + [rootNode.name]: [toITaskData([{ data: { value: 2 } }])], + [subNode.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([node2])); + + // ASSERT + expect(newRunData).toEqual({ + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }); + }); + + // ► + // ┌─────┐ ┌────────┐ ┌────────┐ + // │node1├──►rootNode├──►rootNode│ + // └─────┘ └───▲────┘ └───▲────┘ + // │ │ + // │ ┌───┴───┐ + // └───────┤subNode│ + // └───────┘ + test('removes run data of sub nodes as well if the sub node is shared between multiple root nodes', () => { + // ARRANGE + const node1 = createNodeData({ name: 'Node1' }); + const rootNode1 = createNodeData({ name: 'Root Node 1' }); + const rootNode2 = createNodeData({ name: 'Root Node 2' }); + const subNode = createNodeData({ name: 'Sub Node' }); + const graph = new DirectedGraph() + .addNodes(node1, rootNode1, rootNode2, subNode) + .addConnections( + { from: node1, to: rootNode1 }, + { from: rootNode1, to: rootNode2 }, + { from: subNode, to: rootNode1, type: NodeConnectionType.AiLanguageModel }, + { from: subNode, to: rootNode2, type: NodeConnectionType.AiLanguageModel }, + ); + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [rootNode1.name]: [toITaskData([{ data: { value: 1 } }])], + [rootNode2.name]: [toITaskData([{ data: { value: 2 } }])], + [subNode.name]: [toITaskData([{ data: { value: 3 } }])], + }; + + // ACT + const newRunData = cleanRunData(runData, graph, new Set([rootNode1])); + + // ASSERT + expect(newRunData).toEqual({ + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }); + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/directed-graph.test.ts similarity index 99% rename from packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/directed-graph.test.ts index 5d769004a5..a825c144a9 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/directed-graph.test.ts @@ -13,7 +13,7 @@ import type { INode } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; import { createNodeData, defaultWorkflowParameter } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; +import { DirectedGraph } from '../directed-graph'; describe('DirectedGraph', () => { // ┌─────┐ ┌─────┐ ┌─────┐ diff --git a/packages/core/src/PartialExecutionUtils/__tests__/filterDisabledNodes.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/filter-disabled-nodes.test.ts similarity index 97% rename from packages/core/src/PartialExecutionUtils/__tests__/filterDisabledNodes.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/filter-disabled-nodes.test.ts index 7e60e009a8..69348720f7 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/filterDisabledNodes.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/filter-disabled-nodes.test.ts @@ -12,8 +12,8 @@ import { NodeConnectionType } from 'n8n-workflow'; import { createNodeData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { filterDisabledNodes } from '../filterDisabledNodes'; +import { DirectedGraph } from '../directed-graph'; +import { filterDisabledNodes } from '../filter-disabled-nodes'; describe('filterDisabledNodes', () => { // XX diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/find-start-nodes.test.ts similarity index 99% rename from packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/find-start-nodes.test.ts index f2a99fdb92..a0c86515c2 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/find-start-nodes.test.ts @@ -12,8 +12,8 @@ import { type IPinData, type IRunData } from 'n8n-workflow'; import { createNodeData, toITaskData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { findStartNodes, isDirty } from '../findStartNodes'; +import { DirectedGraph } from '../directed-graph'; +import { findStartNodes, isDirty } from '../find-start-nodes'; describe('isDirty', () => { test("if the node has pinned data it's not dirty", () => { diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/find-subgraph.test.ts similarity index 99% rename from packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/find-subgraph.test.ts index e479214e10..1a6382017a 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/find-subgraph.test.ts @@ -12,8 +12,8 @@ import { NodeConnectionType } from 'n8n-workflow'; import { createNodeData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { findSubgraph } from '../findSubgraph'; +import { DirectedGraph } from '../directed-graph'; +import { findSubgraph } from '../find-subgraph'; describe('findSubgraph', () => { // ►► diff --git a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/get-source-data-groups.test.ts similarity index 99% rename from packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/get-source-data-groups.test.ts index d8c3485d65..872a452aa7 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/get-source-data-groups.test.ts @@ -11,8 +11,8 @@ import type { IPinData } from 'n8n-workflow'; import { NodeConnectionType, type IRunData } from 'n8n-workflow'; import { createNodeData, toITaskData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { getSourceDataGroups } from '../getSourceDataGroups'; +import { DirectedGraph } from '../directed-graph'; +import { getSourceDataGroups } from '../get-source-data-groups'; describe('getSourceDataGroups', () => { //┌───────┐1 diff --git a/packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/handle-cycles.test.ts similarity index 98% rename from packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/handle-cycles.test.ts index def9fed0ff..06ed5daf47 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/handleCycles.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/handle-cycles.test.ts @@ -10,8 +10,8 @@ // PD denotes that the node has pinned data import { createNodeData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { handleCycles } from '../handleCycles'; +import { DirectedGraph } from '../directed-graph'; +import { handleCycles } from '../handle-cycles'; describe('handleCycles', () => { // ┌────┐ ┌─────────┐ diff --git a/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/helpers.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/__tests__/helpers.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/helpers.ts diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/recreate-node-execution-stack.test.ts similarity index 99% rename from packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/recreate-node-execution-stack.test.ts index b78b9df135..0f20896e21 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/__tests__/recreate-node-execution-stack.test.ts @@ -18,15 +18,14 @@ import type { } from 'n8n-workflow'; import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; +import { createNodeData, toITaskData } from './helpers'; +import { DirectedGraph } from '../directed-graph'; +import { findSubgraph } from '../find-subgraph'; import { addWaitingExecution, addWaitingExecutionSource, recreateNodeExecutionStack, -} from '@/PartialExecutionUtils/recreateNodeExecutionStack'; - -import { createNodeData, toITaskData } from './helpers'; -import { DirectedGraph } from '../DirectedGraph'; -import { findSubgraph } from '../findSubgraph'; +} from '../recreate-node-execution-stack'; describe('recreateNodeExecutionStack', () => { // ►► diff --git a/packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/to-iconnections.test.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/to-iconnections.test.ts diff --git a/packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts b/packages/core/src/execution-engine/partial-execution-utils/__tests__/to-itask-data.test.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts rename to packages/core/src/execution-engine/partial-execution-utils/__tests__/to-itask-data.test.ts diff --git a/packages/core/src/PartialExecutionUtils/cleanRunData.ts b/packages/core/src/execution-engine/partial-execution-utils/clean-run-data.ts similarity index 53% rename from packages/core/src/PartialExecutionUtils/cleanRunData.ts rename to packages/core/src/execution-engine/partial-execution-utils/clean-run-data.ts index 6ed5db6100..46f6eea52d 100644 --- a/packages/core/src/PartialExecutionUtils/cleanRunData.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/clean-run-data.ts @@ -1,6 +1,6 @@ -import type { INode, IRunData } from 'n8n-workflow'; +import { NodeConnectionType, type INode, type IRunData } from 'n8n-workflow'; -import type { DirectedGraph } from './DirectedGraph'; +import type { DirectedGraph } from './directed-graph'; /** * Returns new run data that does not contain data for any node that is a child @@ -16,10 +16,22 @@ export function cleanRunData( for (const startNode of startNodes) { delete newRunData[startNode.name]; - const children = graph.getChildren(startNode); - for (const child of children) { - delete newRunData[child.name]; + const children = graph.getChildren(startNode); + for (const node of [startNode, ...children]) { + delete newRunData[node.name]; + + // Delete runData for subNodes + const subNodeConnections = graph.getParentConnections(node); + for (const subNodeConnection of subNodeConnections) { + // Sub nodes never use the Main connection type, so this filters out + // the connection that goes upstream of the startNode. + if (subNodeConnection.type === NodeConnectionType.Main) { + continue; + } + + delete newRunData[subNodeConnection.from.name]; + } } } diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/execution-engine/partial-execution-utils/directed-graph.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/DirectedGraph.ts rename to packages/core/src/execution-engine/partial-execution-utils/directed-graph.ts diff --git a/packages/core/src/PartialExecutionUtils/filterDisabledNodes.ts b/packages/core/src/execution-engine/partial-execution-utils/filter-disabled-nodes.ts similarity index 88% rename from packages/core/src/PartialExecutionUtils/filterDisabledNodes.ts rename to packages/core/src/execution-engine/partial-execution-utils/filter-disabled-nodes.ts index c381f70759..af9ffb3512 100644 --- a/packages/core/src/PartialExecutionUtils/filterDisabledNodes.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/filter-disabled-nodes.ts @@ -1,6 +1,6 @@ import { NodeConnectionType } from 'n8n-workflow'; -import type { DirectedGraph } from './DirectedGraph'; +import type { DirectedGraph } from './directed-graph'; export function filterDisabledNodes(graph: DirectedGraph): DirectedGraph { const filteredGraph = graph.clone(); diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/execution-engine/partial-execution-utils/find-start-nodes.ts similarity index 98% rename from packages/core/src/PartialExecutionUtils/findStartNodes.ts rename to packages/core/src/execution-engine/partial-execution-utils/find-start-nodes.ts index 1c1c0b9fc7..d7b3fb522d 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/find-start-nodes.ts @@ -1,7 +1,7 @@ import { NodeConnectionType, type INode, type IPinData, type IRunData } from 'n8n-workflow'; -import type { DirectedGraph } from './DirectedGraph'; -import { getIncomingData, getIncomingDataFromAnyRun } from './getIncomingData'; +import type { DirectedGraph } from './directed-graph'; +import { getIncomingData, getIncomingDataFromAnyRun } from './get-incoming-data'; /** * A node is dirty if either of the following is true: diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/execution-engine/partial-execution-utils/find-subgraph.ts similarity index 97% rename from packages/core/src/PartialExecutionUtils/findSubgraph.ts rename to packages/core/src/execution-engine/partial-execution-utils/find-subgraph.ts index 039124c1ad..f333f4764e 100644 --- a/packages/core/src/PartialExecutionUtils/findSubgraph.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/find-subgraph.ts @@ -1,7 +1,7 @@ import { NodeConnectionType, type INode } from 'n8n-workflow'; -import type { GraphConnection } from './DirectedGraph'; -import { DirectedGraph } from './DirectedGraph'; +import type { GraphConnection } from './directed-graph'; +import { DirectedGraph } from './directed-graph'; function findSubgraphRecursive( graph: DirectedGraph, diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/execution-engine/partial-execution-utils/find-trigger-for-partial-execution.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts rename to packages/core/src/execution-engine/partial-execution-utils/find-trigger-for-partial-execution.ts diff --git a/packages/core/src/PartialExecutionUtils/getIncomingData.ts b/packages/core/src/execution-engine/partial-execution-utils/get-incoming-data.ts similarity index 100% rename from packages/core/src/PartialExecutionUtils/getIncomingData.ts rename to packages/core/src/execution-engine/partial-execution-utils/get-incoming-data.ts diff --git a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts b/packages/core/src/execution-engine/partial-execution-utils/get-source-data-groups.ts similarity index 98% rename from packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts rename to packages/core/src/execution-engine/partial-execution-utils/get-source-data-groups.ts index d9a9940816..d96aba3062 100644 --- a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/get-source-data-groups.ts @@ -1,6 +1,6 @@ import { type INode, type IPinData, type IRunData } from 'n8n-workflow'; -import type { GraphConnection, DirectedGraph } from './DirectedGraph'; +import type { GraphConnection, DirectedGraph } from './directed-graph'; function sortByInputIndexThenByName( connection1: GraphConnection, diff --git a/packages/core/src/PartialExecutionUtils/handleCycles.ts b/packages/core/src/execution-engine/partial-execution-utils/handle-cycles.ts similarity index 96% rename from packages/core/src/PartialExecutionUtils/handleCycles.ts rename to packages/core/src/execution-engine/partial-execution-utils/handle-cycles.ts index 94a8ae8cbc..1e52b491ed 100644 --- a/packages/core/src/PartialExecutionUtils/handleCycles.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/handle-cycles.ts @@ -1,7 +1,7 @@ import type { INode } from 'n8n-workflow'; import * as a from 'node:assert/strict'; -import type { DirectedGraph } from './DirectedGraph'; +import type { DirectedGraph } from './directed-graph'; /** * Returns a new set of start nodes. diff --git a/packages/core/src/execution-engine/partial-execution-utils/index.ts b/packages/core/src/execution-engine/partial-execution-utils/index.ts new file mode 100644 index 0000000000..d363f52302 --- /dev/null +++ b/packages/core/src/execution-engine/partial-execution-utils/index.ts @@ -0,0 +1,8 @@ +export { DirectedGraph } from './directed-graph'; +export { findTriggerForPartialExecution } from './find-trigger-for-partial-execution'; +export { findStartNodes } from './find-start-nodes'; +export { findSubgraph } from './find-subgraph'; +export { recreateNodeExecutionStack } from './recreate-node-execution-stack'; +export { cleanRunData } from './clean-run-data'; +export { handleCycles } from './handle-cycles'; +export { filterDisabledNodes } from './filter-disabled-nodes'; diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/execution-engine/partial-execution-utils/recreate-node-execution-stack.ts similarity index 97% rename from packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts rename to packages/core/src/execution-engine/partial-execution-utils/recreate-node-execution-stack.ts index 542d4b8fbd..95aced2515 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/execution-engine/partial-execution-utils/recreate-node-execution-stack.ts @@ -12,9 +12,9 @@ import { type IWaitingForExecutionSource, } from 'n8n-workflow'; -import type { DirectedGraph } from './DirectedGraph'; -import { getIncomingDataFromAnyRun } from './getIncomingData'; -import { getSourceDataGroups } from './getSourceDataGroups'; +import type { DirectedGraph } from './directed-graph'; +import { getIncomingDataFromAnyRun } from './get-incoming-data'; +import { getSourceDataGroups } from './get-source-data-groups'; export function addWaitingExecution( waitingExecution: IWaitingForExecution, diff --git a/packages/core/src/RoutingNode.ts b/packages/core/src/execution-engine/routing-node.ts similarity index 89% rename from packages/core/src/RoutingNode.ts rename to packages/core/src/execution-engine/routing-node.ts index 1c041735d8..e62bdfd34c 100644 --- a/packages/core/src/RoutingNode.ts +++ b/packages/core/src/execution-engine/routing-node.ts @@ -6,24 +6,25 @@ import get from 'lodash/get'; import merge from 'lodash/merge'; import set from 'lodash/set'; -import { NodeHelpers, NodeApiError, NodeOperationError, sleep } from 'n8n-workflow'; +import { + NodeHelpers, + NodeApiError, + NodeOperationError, + sleep, + NodeConnectionType, +} from 'n8n-workflow'; import type { ICredentialDataDecryptedObject, ICredentialsDecrypted, IHttpRequestOptions, IN8nHttpFullResponse, - INode, INodeExecutionData, INodeParameters, INodePropertyOptions, INodeType, DeclarativeRestApiSettings, - IRunExecutionData, - ITaskDataConnections, IWorkflowDataProxyAdditionalKeys, - IWorkflowExecuteAdditionalData, NodeParameterValue, - WorkflowExecuteMode, IDataObject, IExecuteData, IExecuteSingleFunctions, @@ -33,70 +34,39 @@ import type { NodeParameterValueType, PostReceiveAction, JsonObject, - CloseFunction, INodeCredentialDescription, IExecutePaginationFunctions, - Workflow, } from 'n8n-workflow'; import url from 'node:url'; -import { ExecuteContext, ExecuteSingleContext } from './node-execution-context'; +import { type ExecuteContext, ExecuteSingleContext } from './node-execution-context'; export class RoutingNode { - additionalData: IWorkflowExecuteAdditionalData; - - connectionInputData: INodeExecutionData[]; - - node: INode; - - mode: WorkflowExecuteMode; - - runExecutionData: IRunExecutionData; - - workflow: Workflow; - constructor( - workflow: Workflow, - node: INode, - connectionInputData: INodeExecutionData[], - runExecutionData: IRunExecutionData, - additionalData: IWorkflowExecuteAdditionalData, - mode: WorkflowExecuteMode, - ) { - this.additionalData = additionalData; - this.connectionInputData = connectionInputData; - this.runExecutionData = runExecutionData; - this.mode = mode; - this.node = node; - this.workflow = workflow; - } + private readonly context: ExecuteContext, + private readonly nodeType: INodeType, + private readonly credentialsDecrypted?: ICredentialsDecrypted, + ) {} // eslint-disable-next-line complexity - async runNode( - inputData: ITaskDataConnections, - runIndex: number, - nodeType: INodeType, - executeData: IExecuteData, - credentialsDecrypted?: ICredentialsDecrypted, - abortSignal?: AbortSignal, - ): Promise { - const items = inputData.main[0] as INodeExecutionData[]; - const returnData: INodeExecutionData[] = []; - - const closeFunctions: CloseFunction[] = []; - const executeFunctions = new ExecuteContext( - this.workflow, - this.node, - this.additionalData, - this.mode, - this.runExecutionData, - runIndex, - this.connectionInputData, - inputData, + async runNode(): Promise { + const { context, nodeType, credentialsDecrypted } = this; + const { + additionalData, executeData, - closeFunctions, - abortSignal, - ); + inputData, + node, + workflow, + mode, + runIndex, + connectionInputData, + runExecutionData, + } = context; + const abortSignal = context.getExecutionCancelSignal(); + + const items = (inputData[NodeConnectionType.Main] ?? + inputData[NodeConnectionType.AiTool])[0] as INodeExecutionData[]; + const returnData: INodeExecutionData[] = []; let credentialDescription: INodeCredentialDescription | undefined; @@ -104,17 +74,14 @@ export class RoutingNode { if (nodeType.description.credentials.length === 1) { credentialDescription = nodeType.description.credentials[0]; } else { - const authenticationMethod = executeFunctions.getNodeParameter( - 'authentication', - 0, - ) as string; + const authenticationMethod = context.getNodeParameter('authentication', 0) as string; credentialDescription = nodeType.description.credentials.find((x) => x.displayOptions?.show?.authentication?.includes(authenticationMethod), ); if (!credentialDescription) { throw new NodeOperationError( - this.node, - `Node type "${this.node.type}" does not have any credentials of type "${authenticationMethod}" defined`, + node, + `Node type "${node.type}" does not have any credentials of type "${authenticationMethod}" defined`, { level: 'warning' }, ); } @@ -127,7 +94,7 @@ export class RoutingNode { } else if (credentialDescription) { try { credentials = - (await executeFunctions.getCredentials( + (await context.getCredentials( credentialDescription.name, 0, )) || {}; @@ -142,7 +109,7 @@ export class RoutingNode { } } - const { batching } = executeFunctions.getNodeParameter('requestOptions', 0, {}) as { + const { batching } = context.getNodeParameter('requestOptions', 0, {}) as { batching: { batch: { batchSize: number; batchInterval: number } }; }; @@ -163,13 +130,13 @@ export class RoutingNode { } const thisArgs = new ExecuteSingleContext( - this.workflow, - this.node, - this.additionalData, - this.mode, - this.runExecutionData, + workflow, + node, + additionalData, + mode, + runExecutionData, runIndex, - this.connectionInputData, + connectionInputData, inputData, itemIndex, executeData, @@ -214,7 +181,7 @@ export class RoutingNode { itemIndex, runIndex, executeData, - { $credentials: credentials, $version: this.node.typeVersion }, + { $credentials: credentials, $version: node.typeVersion }, false, ) as string; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -223,14 +190,14 @@ export class RoutingNode { } for (const property of nodeType.description.properties) { - let value = get(this.node.parameters, property.name, []) as string | NodeParameterValue; + let value = get(node.parameters, property.name, []) as string | NodeParameterValue; // If the value is an expression resolve it value = this.getParameterValue( value, itemIndex, runIndex, executeData, - { $credentials: credentials, $version: this.node.typeVersion }, + { $credentials: credentials, $version: node.typeVersion }, false, ) as string | NodeParameterValue; @@ -240,7 +207,7 @@ export class RoutingNode { itemIndex, runIndex, '', - { $credentials: credentials, $value: value, $version: this.node.typeVersion }, + { $credentials: credentials, $value: value, $version: node.typeVersion }, ); this.mergeOptions(itemContext[itemIndex].requestData, tempOptions); @@ -255,7 +222,7 @@ export class RoutingNode { !(property in proxyParsed) || proxyParsed[property as keyof typeof proxyParsed] === null ) { - throw new NodeOperationError(this.node, 'The proxy is not value', { + throw new NodeOperationError(node, 'The proxy is not value', { runIndex, itemIndex, description: `The proxy URL does not contain a valid value for "${property}"`, @@ -291,7 +258,7 @@ export class RoutingNode { } requestPromises.push( - this.makeRoutingRequest( + this.makeRequest( itemContext[itemIndex].requestData, itemContext[itemIndex].thisArgs, itemIndex, @@ -327,7 +294,7 @@ export class RoutingNode { throw error; } - throw new NodeApiError(this.node, error as JsonObject, { + throw new NodeApiError(node, error as JsonObject, { runIndex, itemIndex, message: error?.message, @@ -347,7 +314,7 @@ export class RoutingNode { return [returnData]; } - mergeOptions( + private mergeOptions( destinationOptions: DeclarativeRestApiSettings.ResultOptions, sourceOptions?: DeclarativeRestApiSettings.ResultOptions, ): void { @@ -368,7 +335,7 @@ export class RoutingNode { } } - async runPostReceiveAction( + private async runPostReceiveAction( executeSingleFunctions: IExecuteSingleFunctions, action: PostReceiveAction, inputData: INodeExecutionData[], @@ -380,6 +347,9 @@ export class RoutingNode { if (typeof action === 'function') { return await action.call(executeSingleFunctions, inputData, responseData); } + + const { node } = this.context; + if (action.type === 'rootProperty') { try { return inputData.flatMap((item) => { @@ -395,13 +365,14 @@ export class RoutingNode { }); }); } catch (error) { - throw new NodeOperationError(this.node, error as Error, { + throw new NodeOperationError(node, error as Error, { runIndex, itemIndex, description: `The rootProperty "${action.properties.property}" could not be found on item.`, }); } } + if (action.type === 'filter') { const passValue = action.properties.pass; @@ -416,7 +387,7 @@ export class RoutingNode { $response: responseData, $responseItem: item.json, $value: parameterValue, - $version: this.node.typeVersion, + $version: node.typeVersion, }, false, ) as boolean; @@ -424,17 +395,19 @@ export class RoutingNode { return inputData; } + if (action.type === 'limit') { const maxResults = this.getParameterValue( action.properties.maxResults, itemIndex, runIndex, executeSingleFunctions.getExecuteData(), - { $response: responseData, $value: parameterValue, $version: this.node.typeVersion }, + { $response: responseData, $value: parameterValue, $version: node.typeVersion }, false, ) as string; return inputData.slice(0, parseInt(maxResults, 10)); } + if (action.type === 'set') { const { value } = action.properties; // If the value is an expression resolve it @@ -445,12 +418,13 @@ export class RoutingNode { itemIndex, runIndex, executeSingleFunctions.getExecuteData(), - { $response: responseData, $value: parameterValue, $version: this.node.typeVersion }, + { $response: responseData, $value: parameterValue, $version: node.typeVersion }, false, ) as IDataObject, }, ]; } + if (action.type === 'sort') { // Sort the returned options const sortKey = action.properties.key; @@ -468,6 +442,7 @@ export class RoutingNode { return inputData; } + if (action.type === 'setKeyValue') { const returnData: INodeExecutionData[] = []; @@ -491,7 +466,7 @@ export class RoutingNode { $response: responseData, $responseItem: item.json, $value: parameterValue, - $version: this.node.typeVersion, + $version: node.typeVersion, }, false, ) as string; @@ -503,6 +478,7 @@ export class RoutingNode { return returnData; } + if (action.type === 'binaryData') { const body = (responseData.body = Buffer.from(responseData.body as string)); let { destinationProperty } = action.properties; @@ -512,7 +488,7 @@ export class RoutingNode { itemIndex, runIndex, executeSingleFunctions.getExecuteData(), - { $response: responseData, $value: parameterValue, $version: this.node.typeVersion }, + { $response: responseData, $value: parameterValue, $version: node.typeVersion }, false, ) as string; @@ -535,7 +511,7 @@ export class RoutingNode { return []; } - async postProcessResponseData( + private async postProcessResponseData( executeSingleFunctions: IExecuteSingleFunctions, responseData: IN8nHttpFullResponse, requestData: DeclarativeRestApiSettings.ResultOptions, @@ -580,7 +556,7 @@ export class RoutingNode { return returnData; } - async rawRoutingRequest( + private async rawRoutingRequest( executeSingleFunctions: IExecuteSingleFunctions, requestData: DeclarativeRestApiSettings.ResultOptions, credentialType?: string, @@ -604,7 +580,7 @@ export class RoutingNode { return responseData; } - async makeRoutingRequest( + private async makeRequest( requestData: DeclarativeRestApiSettings.ResultOptions, executeSingleFunctions: IExecuteSingleFunctions, itemIndex: number, @@ -639,6 +615,7 @@ export class RoutingNode { ); }; + const { node } = this.context; const executePaginationFunctions = Object.create(executeSingleFunctions, { makeRoutingRequest: { value: makeRoutingRequest }, }) as IExecutePaginationFunctions; @@ -669,7 +646,7 @@ export class RoutingNode { const additionalKeys = { $request: requestData.options, $response: {} as IN8nHttpFullResponse, - $version: this.node.typeVersion, + $version: node.typeVersion, }; do { @@ -763,7 +740,7 @@ export class RoutingNode { | undefined; if (tempResponseValue === undefined) { throw new NodeOperationError( - this.node, + node, `The rootProperty "${properties.rootProperty}" could not be found on item.`, { runIndex, itemIndex }, ); @@ -801,7 +778,7 @@ export class RoutingNode { return responseData; } - getParameterValue( + private getParameterValue( parameterValue: NodeParameterValueType, itemIndex: number, runIndex: number, @@ -813,14 +790,15 @@ export class RoutingNode { typeof parameterValue === 'object' || (typeof parameterValue === 'string' && parameterValue.charAt(0) === '=') ) { - return this.workflow.expression.getParameterValue( + const { node, workflow, mode, connectionInputData, runExecutionData } = this.context; + return workflow.expression.getParameterValue( parameterValue, - this.runExecutionData ?? null, + runExecutionData ?? null, runIndex, itemIndex, - this.node.name, - this.connectionInputData, - this.mode, + node.name, + connectionInputData, + mode, additionalKeys ?? {}, executeData, returnObjectAsString, @@ -851,14 +829,8 @@ export class RoutingNode { }; let basePath = path ? `${path}.` : ''; - if ( - !NodeHelpers.displayParameter( - this.node.parameters, - nodeProperties, - this.node, - this.node.parameters, - ) - ) { + const { node } = this.context; + if (!NodeHelpers.displayParameter(node.parameters, nodeProperties, node, node.parameters)) { return undefined; } if (nodeProperties.routing) { @@ -1032,7 +1004,7 @@ export class RoutingNode { let value; if (nodeProperties.type === 'options') { const optionValue = NodeHelpers.getParameterValueByPath( - this.node.parameters, + node.parameters, nodeProperties.name, basePath.slice(0, -1), ); @@ -1050,14 +1022,14 @@ export class RoutingNode { itemIndex, runIndex, `${basePath}${nodeProperties.name}`, - { $value: optionValue, $version: this.node.typeVersion }, + { $value: optionValue, $version: node.typeVersion }, ); this.mergeOptions(returnData, tempOptions); } } else if (nodeProperties.type === 'collection') { value = NodeHelpers.getParameterValueByPath( - this.node.parameters, + node.parameters, nodeProperties.name, basePath.slice(0, -1), ); @@ -1074,7 +1046,7 @@ export class RoutingNode { itemIndex, runIndex, `${basePath}${nodeProperties.name}`, - { $version: this.node.typeVersion }, + { $version: node.typeVersion }, ); this.mergeOptions(returnData, tempOptions); @@ -1085,7 +1057,7 @@ export class RoutingNode { for (const propertyOptions of nodeProperties.options as INodePropertyCollection[]) { // Check if the option got set and if not skip it value = NodeHelpers.getParameterValueByPath( - this.node.parameters, + node.parameters, propertyOptions.name, basePath.slice(0, -1), ); diff --git a/packages/core/src/ScheduledTaskManager.ts b/packages/core/src/execution-engine/scheduled-task-manager.ts similarity index 95% rename from packages/core/src/ScheduledTaskManager.ts rename to packages/core/src/execution-engine/scheduled-task-manager.ts index 0c33f9872c..1a20700b00 100644 --- a/packages/core/src/ScheduledTaskManager.ts +++ b/packages/core/src/execution-engine/scheduled-task-manager.ts @@ -2,7 +2,7 @@ import { Service } from '@n8n/di'; import { CronJob } from 'cron'; import type { CronExpression, Workflow } from 'n8n-workflow'; -import { InstanceSettings } from './InstanceSettings'; +import { InstanceSettings } from '@/instance-settings'; @Service() export class ScheduledTaskManager { diff --git a/packages/core/src/SSHClientsManager.ts b/packages/core/src/execution-engine/ssh-clients-manager.ts similarity index 100% rename from packages/core/src/SSHClientsManager.ts rename to packages/core/src/execution-engine/ssh-clients-manager.ts diff --git a/packages/core/src/TriggersAndPollers.ts b/packages/core/src/execution-engine/triggers-and-pollers.ts similarity index 68% rename from packages/core/src/TriggersAndPollers.ts rename to packages/core/src/execution-engine/triggers-and-pollers.ts index 681d50641a..9c90327149 100644 --- a/packages/core/src/TriggersAndPollers.ts +++ b/packages/core/src/execution-engine/triggers-and-pollers.ts @@ -5,7 +5,6 @@ import type { INode, INodeExecutionData, IPollFunctions, - IGetExecuteTriggerFunctions, IWorkflowExecuteAdditionalData, WorkflowExecuteMode, WorkflowActivateMode, @@ -14,6 +13,9 @@ import type { IExecuteResponsePromiseData, IRun, } from 'n8n-workflow'; +import assert from 'node:assert'; + +import type { IGetExecuteTriggerFunctions } from './interfaces'; @Service() export class TriggersAndPollers { @@ -46,46 +48,34 @@ export class TriggersAndPollers { // Add the manual trigger response which resolves when the first time data got emitted triggerResponse!.manualTriggerResponse = new Promise((resolve, reject) => { + const { hooks } = additionalData; + assert.ok(hooks, 'Execution lifecycle hooks are not defined'); + triggerFunctions.emit = ( - (resolveEmit) => - ( - data: INodeExecutionData[][], - responsePromise?: IDeferredPromise, - donePromise?: IDeferredPromise, - ) => { - additionalData.hooks!.hookFunctions.sendResponse = [ - async (response: IExecuteResponsePromiseData): Promise => { - if (responsePromise) { - responsePromise.resolve(response); - } - }, - ]; - - if (donePromise) { - additionalData.hooks!.hookFunctions.workflowExecuteAfter?.unshift( - async (runData: IRun): Promise => { - return donePromise.resolve(runData); - }, - ); - } - - resolveEmit(data); + data: INodeExecutionData[][], + responsePromise?: IDeferredPromise, + donePromise?: IDeferredPromise, + ) => { + if (responsePromise) { + hooks.addHandler('sendResponse', (response) => responsePromise.resolve(response)); } - )(resolve); + + if (donePromise) { + hooks.addHandler('workflowExecuteAfter', (runData) => donePromise.resolve(runData)); + } + + resolve(data); + }; + triggerFunctions.emitError = ( - (rejectEmit) => - (error: Error, responsePromise?: IDeferredPromise) => { - additionalData.hooks!.hookFunctions.sendResponse = [ - async (): Promise => { - if (responsePromise) { - responsePromise.reject(error); - } - }, - ]; - - rejectEmit(error); + error: Error, + responsePromise?: IDeferredPromise, + ) => { + if (responsePromise) { + hooks.addHandler('sendResponse', () => responsePromise.reject(error)); } - )(reject); + reject(error); + }; }); return triggerResponse; diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/execution-engine/workflow-execute.ts similarity index 97% rename from packages/core/src/WorkflowExecute.ts rename to packages/core/src/execution-engine/workflow-execute.ts index 3817fe32b2..00a7a580a3 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/execution-engine/workflow-execute.ts @@ -44,7 +44,6 @@ import type { } from 'n8n-workflow'; import { LoggerProxy as Logger, - WorkflowOperationError, NodeHelpers, NodeConnectionType, ApplicationError, @@ -55,9 +54,11 @@ import { } from 'n8n-workflow'; import PCancelable from 'p-cancelable'; -import { ErrorReporter } from './error-reporter'; +import { ErrorReporter } from '@/errors/error-reporter'; +import { WorkflowHasIssuesError } from '@/errors/workflow-has-issues.error'; +import * as NodeExecuteFunctions from '@/node-execute-functions'; + import { ExecuteContext, PollContext } from './node-execution-context'; -import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import { DirectedGraph, findStartNodes, @@ -67,9 +68,9 @@ import { recreateNodeExecutionStack, handleCycles, filterDisabledNodes, -} from './PartialExecutionUtils'; -import { RoutingNode } from './RoutingNode'; -import { TriggersAndPollers } from './TriggersAndPollers'; +} from './partial-execution-utils'; +import { RoutingNode } from './routing-node'; +import { TriggersAndPollers } from './triggers-and-pollers'; export class WorkflowExecute { private status: ExecutionStatus = 'new'; @@ -404,19 +405,6 @@ export class WorkflowExecute { return this.processRunExecutionData(graph.toWorkflow({ ...workflow })); } - /** - * Executes the hook with the given name - * - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async executeHook(hookName: string, parameters: any[]): Promise { - if (this.additionalData.hooks === undefined) { - return; - } - - return await this.additionalData.hooks.executeHookFunctions(hookName, parameters); - } - /** * Merges temporary execution metadata into the final runData structure. * During workflow execution, metadata is collected in a temporary location @@ -1170,27 +1158,23 @@ export class WorkflowExecute { // For webhook nodes always simply pass the data through return { data: inputData.main as INodeExecutionData[][] }; } else { - // For nodes which have routing information on properties - - const routingNode = new RoutingNode( + // NOTE: This block is only called by nodes tests. + // In the application, declarative nodes get assigned a `.execute` method in NodeTypes. + const context = new ExecuteContext( workflow, node, - connectionInputData, - runExecutionData ?? null, additionalData, mode, + runExecutionData, + runIndex, + connectionInputData, + inputData, + executionData, + [], ); - - return { - data: await routingNode.runNode( - inputData, - runIndex, - nodeType, - executionData, - undefined, - abortSignal, - ), - }; + const routingNode = new RoutingNode(context, nodeType); + const data = await routingNode.runNode(); + return { data }; } } @@ -1210,7 +1194,21 @@ export class WorkflowExecute { this.status = 'running'; - const startNode = this.runExecutionData.executionData!.nodeExecutionStack[0].node.name; + const { hooks, executionId } = this.additionalData; + assert.ok(hooks, 'Failed to run workflow due to missing execution lifecycle hooks'); + + if (!this.runExecutionData.executionData) { + throw new ApplicationError('Failed to run workflow due to missing execution data', { + extra: { + workflowId: workflow.id, + executionId, + mode: this.mode, + }, + }); + } + + /** Node execution stack will be empty for an execution containing only Chat Trigger. */ + const startNode = this.runExecutionData.executionData.nodeExecutionStack.at(0)?.node.name; let destinationNode: string | undefined; if (this.runExecutionData.startData && this.runExecutionData.startData.destinationNode) { @@ -1225,9 +1223,7 @@ export class WorkflowExecute { pinDataNodeNames, }); if (workflowIssues !== null) { - throw new WorkflowOperationError( - 'The workflow has issues and cannot be executed for that reason. Please fix them first.', - ); + throw new WorkflowHasIssuesError(); } // Variables which hold temporary data for each node-execution @@ -1245,7 +1241,7 @@ export class WorkflowExecute { if (this.runExecutionData.waitTill) { const lastNodeExecuted = this.runExecutionData.resultData.lastNodeExecuted as string; - this.runExecutionData.executionData!.nodeExecutionStack[0].node.disabled = true; + this.runExecutionData.executionData.nodeExecutionStack[0].node.disabled = true; this.runExecutionData.waitTill = undefined; this.runExecutionData.resultData.runData[lastNodeExecuted].pop(); } @@ -1263,14 +1259,14 @@ export class WorkflowExecute { this.status = 'canceled'; this.abortController.abort(); const fullRunData = this.getFullRunData(startedAt); - void this.executeHook('workflowExecuteAfter', [fullRunData]); + void hooks.runHook('workflowExecuteAfter', [fullRunData]); }); // eslint-disable-next-line complexity const returnPromise = (async () => { try { if (!this.additionalData.restartExecutionId) { - await this.executeHook('workflowExecuteBefore', [workflow, this.runExecutionData]); + await hooks.runHook('workflowExecuteBefore', [workflow, this.runExecutionData]); } } catch (error) { const e = error as unknown as ExecutionBaseError; @@ -1354,7 +1350,7 @@ export class WorkflowExecute { node: executionNode.name, workflowId: workflow.id, }); - await this.executeHook('nodeExecuteBefore', [executionNode.name]); + await hooks.runHook('nodeExecuteBefore', [executionNode.name]); // Get the index of the current run runIndex = 0; @@ -1645,7 +1641,7 @@ export class WorkflowExecute { this.runExecutionData.executionData!.nodeExecutionStack.unshift(executionData); // Only execute the nodeExecuteAfter hook if the node did not get aborted if (!this.isCancelled) { - await this.executeHook('nodeExecuteAfter', [ + await hooks.runHook('nodeExecuteAfter', [ executionNode.name, taskData, this.runExecutionData, @@ -1687,7 +1683,7 @@ export class WorkflowExecute { this.runExecutionData.resultData.runData[executionNode.name].push(taskData); if (this.runExecutionData.waitTill) { - await this.executeHook('nodeExecuteAfter', [ + await hooks.runHook('nodeExecuteAfter', [ executionNode.name, taskData, this.runExecutionData, @@ -1706,7 +1702,7 @@ export class WorkflowExecute { ) { // Before stopping, make sure we are executing hooks so // That frontend is notified for example for manual executions. - await this.executeHook('nodeExecuteAfter', [ + await hooks.runHook('nodeExecuteAfter', [ executionNode.name, taskData, this.runExecutionData, @@ -1816,7 +1812,7 @@ export class WorkflowExecute { // Execute hooks now to make sure that all hooks are executed properly // Await is needed to make sure that we don't fall into concurrency problems // When saving node execution data - await this.executeHook('nodeExecuteAfter', [ + await hooks.runHook('nodeExecuteAfter', [ executionNode.name, taskData, this.runExecutionData, @@ -2019,7 +2015,7 @@ export class WorkflowExecute { this.moveNodeMetadata(); - await this.executeHook('workflowExecuteAfter', [fullRunData, newStaticData]).catch( + await hooks.runHook('workflowExecuteAfter', [fullRunData, newStaticData]).catch( // eslint-disable-next-line @typescript-eslint/no-shadow (error) => { console.error('There was a problem running hook "workflowExecuteAfter"', error); @@ -2112,7 +2108,10 @@ export class WorkflowExecute { this.moveNodeMetadata(); // Prevent from running the hook if the error is an abort error as it was already handled if (!this.isCancelled) { - await this.executeHook('workflowExecuteAfter', [fullRunData, newStaticData]); + await this.additionalData.hooks?.runHook('workflowExecuteAfter', [ + fullRunData, + newStaticData, + ]); } if (closeFunction) { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7abbd9ad9a..0a620468bf 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,29 +1,19 @@ -import * as NodeExecuteFunctions from './NodeExecuteFunctions'; +import * as NodeExecuteFunctions from './node-execute-functions'; -export * from './decorators'; -export * from './errors'; -export * from './ActiveWorkflows'; -export * from './BinaryData/BinaryData.service'; -export * from './BinaryData/types'; -export { Cipher } from './Cipher'; -export * from './Constants'; -export * from './Credentials'; -export * from './DirectoryLoader'; -export * from './Interfaces'; -export { InstanceSettings, InstanceType } from './InstanceSettings'; -export { Logger } from './logging/logger'; -export * from './NodeExecuteFunctions'; -export * from './RoutingNode'; -export * from './WorkflowExecute'; -export { NodeExecuteFunctions }; +export * from './binary-data'; +export * from './constants'; +export * from './credentials'; export * from './data-deduplication-service'; +export * from './decorators'; +export * from './encryption'; export * from './errors'; -export { ObjectStoreService } from './ObjectStore/ObjectStore.service.ee'; -export { BinaryData } from './BinaryData/types'; -export { isStoredMode as isValidNonDefaultMode } from './BinaryData/utils'; -export * from './ExecutionMetadata'; -export * from './node-execution-context'; -export * from './PartialExecutionUtils'; -export { ErrorReporter } from './error-reporter'; -export * from './SerializedBuffer'; -export { isObjectLiteral } from './utils'; +export * from './execution-engine'; +export * from './instance-settings'; +export * from './logging'; +export * from './nodes-loader'; +export * from './utils'; +export { WorkflowHasIssuesError } from './errors/workflow-has-issues.error'; + +export * from './interfaces'; +export * from './node-execute-functions'; +export { NodeExecuteFunctions }; diff --git a/packages/core/test/InstanceSettings.test.ts b/packages/core/src/instance-settings/__tests__/instance-settings.test.ts similarity index 81% rename from packages/core/test/InstanceSettings.test.ts rename to packages/core/src/instance-settings/__tests__/instance-settings.test.ts index 1fe96d3490..b57ed99f09 100644 --- a/packages/core/test/InstanceSettings.test.ts +++ b/packages/core/src/instance-settings/__tests__/instance-settings.test.ts @@ -2,11 +2,10 @@ import { mock } from 'jest-mock-extended'; jest.mock('node:fs', () => mock()); import * as fs from 'node:fs'; -import { InstanceSettings } from '@/InstanceSettings'; -import { InstanceSettingsConfig } from '@/InstanceSettingsConfig'; +import { InstanceSettings } from '@/instance-settings'; +import { InstanceSettingsConfig } from '@/instance-settings/instance-settings-config'; import { Logger } from '@/logging/logger'; - -import { mockInstance } from './utils'; +import { mockInstance } from '@test/utils'; describe('InstanceSettings', () => { const userFolder = '/test'; @@ -214,33 +213,58 @@ describe('InstanceSettings', () => { }); it('should return true if /.dockerenv exists', () => { - mockFs.existsSync.calledWith('/.dockerenv').mockReturnValueOnce(true); + mockFs.existsSync.mockImplementation((path) => path === '/.dockerenv'); expect(settings.isDocker).toBe(true); expect(mockFs.existsSync).toHaveBeenCalledWith('/.dockerenv'); expect(mockFs.readFileSync).not.toHaveBeenCalledWith('/proc/self/cgroup', 'utf8'); }); - it('should return true if /proc/self/cgroup contains docker', () => { - mockFs.existsSync.calledWith('/.dockerenv').mockReturnValueOnce(false); - mockFs.readFileSync - .calledWith('/proc/self/cgroup', 'utf8') - .mockReturnValueOnce('docker cgroup'); - + it('should return true if /run/.containerenv exists', () => { + mockFs.existsSync.mockImplementation((path) => path === '/run/.containerenv'); expect(settings.isDocker).toBe(true); - expect(mockFs.existsSync).toHaveBeenCalledWith('/.dockerenv'); - expect(mockFs.readFileSync).toHaveBeenCalledWith('/proc/self/cgroup', 'utf8'); + expect(mockFs.existsSync).toHaveBeenCalledWith('/run/.containerenv'); + expect(mockFs.readFileSync).not.toHaveBeenCalledWith('/proc/self/cgroup', 'utf8'); }); + test.each(['docker', 'kubepods', 'containerd'])( + 'should return true if /proc/self/cgroup contains %s', + (str) => { + mockFs.existsSync.mockReturnValueOnce(false); + mockFs.readFileSync.calledWith('/proc/self/cgroup', 'utf8').mockReturnValueOnce(str); + + expect(settings.isDocker).toBe(true); + expect(mockFs.existsSync).toHaveBeenCalledWith('/.dockerenv'); + expect(mockFs.readFileSync).toHaveBeenCalledWith('/proc/self/cgroup', 'utf8'); + }, + ); + + test.each(['docker', 'kubelet', 'containerd'])( + 'should return true if /proc/self/mountinfo contains %s', + (str) => { + mockFs.existsSync.mockReturnValueOnce(false); + mockFs.readFileSync.calledWith('/proc/self/cgroup', 'utf8').mockReturnValueOnce(''); + mockFs.readFileSync.calledWith('/proc/self/mountinfo', 'utf8').mockReturnValueOnce(str); + + expect(settings.isDocker).toBe(true); + expect(mockFs.existsSync).toHaveBeenCalledWith('/.dockerenv'); + expect(mockFs.readFileSync).toHaveBeenCalledWith('/proc/self/cgroup', 'utf8'); + expect(mockFs.readFileSync).toHaveBeenCalledWith('/proc/self/mountinfo', 'utf8'); + }, + ); + it('should return false if no docker indicators are found', () => { mockFs.existsSync.calledWith('/.dockerenv').mockReturnValueOnce(false); mockFs.readFileSync.calledWith('/proc/self/cgroup', 'utf8').mockReturnValueOnce(''); + mockFs.readFileSync.calledWith('/proc/self/mountinfo', 'utf8').mockReturnValueOnce(''); expect(settings.isDocker).toBe(false); }); - it('should return false if checking for docker throws an error', () => { - mockFs.existsSync.calledWith('/.dockerenv').mockImplementationOnce(() => { - throw new Error('Access denied'); + it('should return false if reading any of these files throws an error', () => { + mockFs.existsSync.mockReturnValue(false); + mockFs.readFileSync.mockImplementation(() => { + throw new Error('File not found'); }); + expect(settings.isDocker).toBe(false); }); diff --git a/packages/core/src/instance-settings/index.ts b/packages/core/src/instance-settings/index.ts new file mode 100644 index 0000000000..74b05f5d98 --- /dev/null +++ b/packages/core/src/instance-settings/index.ts @@ -0,0 +1 @@ +export { InstanceSettings, InstanceType } from './instance-settings'; diff --git a/packages/core/src/InstanceSettingsConfig.ts b/packages/core/src/instance-settings/instance-settings-config.ts similarity index 100% rename from packages/core/src/InstanceSettingsConfig.ts rename to packages/core/src/instance-settings/instance-settings-config.ts diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/instance-settings/instance-settings.ts similarity index 92% rename from packages/core/src/InstanceSettings.ts rename to packages/core/src/instance-settings/instance-settings.ts index d06f68fc12..5f8a68596b 100644 --- a/packages/core/src/InstanceSettings.ts +++ b/packages/core/src/instance-settings/instance-settings.ts @@ -5,10 +5,10 @@ import { customAlphabet } from 'nanoid'; import { chmodSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import path from 'path'; +import { Memoized } from '@/decorators'; import { Logger } from '@/logging/logger'; -import { Memoized } from './decorators'; -import { InstanceSettingsConfig } from './InstanceSettingsConfig'; +import { InstanceSettingsConfig } from './instance-settings-config'; const nanoid = customAlphabet(ALPHABET, 16); @@ -142,19 +142,30 @@ export class InstanceSettings { } /** - * Whether this instance is running inside a Docker container. - * - * Based on: https://github.com/sindresorhus/is-docker + * Whether this instance is running inside a Docker/Podman/Kubernetes container. */ @Memoized get isDocker() { + if (existsSync('/.dockerenv') || existsSync('/run/.containerenv')) return true; try { - return ( - existsSync('/.dockerenv') || readFileSync('/proc/self/cgroup', 'utf8').includes('docker') - ); - } catch { - return false; - } + const cgroupV1 = readFileSync('/proc/self/cgroup', 'utf8'); + if ( + cgroupV1.includes('docker') || + cgroupV1.includes('kubepods') || + cgroupV1.includes('containerd') + ) + return true; + } catch {} + try { + const cgroupV2 = readFileSync('/proc/self/mountinfo', 'utf8'); + if ( + cgroupV2.includes('docker') || + cgroupV2.includes('kubelet') || + cgroupV2.includes('containerd') + ) + return true; + } catch {} + return false; } update(newSettings: WritableSettings) { diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/interfaces.ts similarity index 73% rename from packages/core/src/Interfaces.ts rename to packages/core/src/interfaces.ts index 2963e46185..a00176eee9 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/interfaces.ts @@ -20,19 +20,4 @@ export interface IWorkflowData { triggerResponses?: ITriggerResponse[]; } -export namespace n8n { - export interface PackageJson { - name: string; - version: string; - n8n?: { - credentials?: string[]; - nodes?: string[]; - }; - author?: { - name?: string; - email?: string; - }; - } -} - export type ExtendedValidationResult = ValidationResult & { fieldName?: string }; diff --git a/packages/core/src/logging/__tests__/logger.test.ts b/packages/core/src/logging/__tests__/logger.test.ts index d34eaf250a..1a9ee8e7ef 100644 --- a/packages/core/src/logging/__tests__/logger.test.ts +++ b/packages/core/src/logging/__tests__/logger.test.ts @@ -7,7 +7,7 @@ import type { GlobalConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; import { LoggerProxy } from 'n8n-workflow'; -import type { InstanceSettingsConfig } from '@/InstanceSettingsConfig'; +import type { InstanceSettingsConfig } from '@/instance-settings/instance-settings-config'; import { Logger } from '../logger'; diff --git a/packages/core/src/logging/index.ts b/packages/core/src/logging/index.ts new file mode 100644 index 0000000000..a1332e91c6 --- /dev/null +++ b/packages/core/src/logging/index.ts @@ -0,0 +1 @@ +export { Logger } from './logger'; diff --git a/packages/core/src/logging/logger.ts b/packages/core/src/logging/logger.ts index 8115f93c86..82a47649ff 100644 --- a/packages/core/src/logging/logger.ts +++ b/packages/core/src/logging/logger.ts @@ -14,9 +14,9 @@ import path, { basename } from 'node:path'; import pc from 'picocolors'; import winston from 'winston'; -import { inDevelopment, inProduction } from '@/Constants'; -import { InstanceSettingsConfig } from '@/InstanceSettingsConfig'; -import { isObjectLiteral } from '@/utils'; +import { inDevelopment, inProduction } from '@/constants'; +import { InstanceSettingsConfig } from '@/instance-settings/instance-settings-config'; +import { isObjectLiteral } from '@/utils/is-object-literal'; const noOp = () => {}; diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/node-execute-functions.ts similarity index 60% rename from packages/core/src/NodeExecuteFunctions.ts rename to packages/core/src/node-execute-functions.ts index 9504ad1c78..34a4757962 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/node-execute-functions.ts @@ -14,31 +14,20 @@ import type { } from '@n8n/client-oauth2'; import { ClientOAuth2 } from '@n8n/client-oauth2'; import { Container } from '@n8n/di'; -import type { AxiosError, AxiosHeaders, AxiosRequestConfig, AxiosResponse } from 'axios'; +import type { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'; import axios from 'axios'; -import chardet from 'chardet'; import crypto, { createHmac } from 'crypto'; -import FileType from 'file-type'; import FormData from 'form-data'; -import { createReadStream } from 'fs'; -import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { IncomingMessage } from 'http'; import { Agent, type AgentOptions } from 'https'; -import iconv from 'iconv-lite'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import merge from 'lodash/merge'; import pick from 'lodash/pick'; -import { extension, lookup } from 'mime-types'; import type { - BinaryHelperFunctions, - FileSystemHelperFunctions, - GenericValue, IAdditionalCredentialOptions, IAllExecuteFunctions, - IBinaryData, ICredentialDataDecryptedObject, - ICredentialTestFunctions, IDataObject, IExecuteData, IExecuteFunctions, @@ -52,13 +41,11 @@ import type { IPollFunctions, IRequestOptions, IRunExecutionData, - ITaskDataConnections, ITriggerFunctions, IWebhookDescription, IWorkflowDataProxyAdditionalKeys, IWorkflowExecuteAdditionalData, NodeExecutionWithMetadata, - NodeHelperFunctions, NodeParameterValueType, PaginationOptions, RequestHelperFunctions, @@ -66,13 +53,6 @@ import type { WorkflowActivateMode, WorkflowExecuteMode, SSHTunnelFunctions, - DeduplicationHelperFunctions, - IDeduplicationOutput, - IDeduplicationOutputItems, - ICheckProcessedOptions, - DeduplicationScope, - DeduplicationItemTypes, - ICheckProcessedContextData, WebhookType, SchedulingFunctions, } from 'n8n-workflow'; @@ -82,7 +62,6 @@ import { NodeOperationError, NodeSslError, deepCopy, - fileTypeFromMimeType, isObjectEmpty, ExecutionBaseError, jsonParse, @@ -91,32 +70,23 @@ import { } from 'n8n-workflow'; import type { Token } from 'oauth-1.0a'; import clientOAuth1 from 'oauth-1.0a'; -import path from 'path'; import { stringify } from 'qs'; import { Readable } from 'stream'; import url, { URL, URLSearchParams } from 'url'; import { Logger } from '@/logging/logger'; -import { BinaryDataService } from './BinaryData/BinaryData.service'; -import type { BinaryData } from './BinaryData/types'; -import { binaryToBuffer } from './BinaryData/utils'; -import { - BINARY_DATA_STORAGE_PATH, - BLOCK_FILE_ACCESS_TO_N8N_FILES, - CONFIG_FILES, - CUSTOM_EXTENSION_ENV, - RESTRICT_FILE_ACCESS_TO, - UM_EMAIL_TEMPLATES_INVITE, - UM_EMAIL_TEMPLATES_PWRESET, -} from './Constants'; -import { DataDeduplicationService } from './data-deduplication-service'; -import { InstanceSettings } from './InstanceSettings'; -import type { IResponseError } from './Interfaces'; // eslint-disable-next-line import/no-cycle -import { PollContext, TriggerContext } from './node-execution-context'; -import { ScheduledTaskManager } from './ScheduledTaskManager'; -import { SSHClientsManager } from './SSHClientsManager'; +import { + binaryToString, + parseIncomingMessage, + parseRequestObject, + PollContext, + TriggerContext, +} from './execution-engine/node-execution-context'; +import { ScheduledTaskManager } from './execution-engine/scheduled-task-manager'; +import { SSHClientsManager } from './execution-engine/ssh-clients-manager'; +import type { IResponseError } from './interfaces'; axios.defaults.timeout = 300000; // Prevent axios from adding x-form-www-urlencoded headers by default @@ -137,31 +107,6 @@ axios.interceptors.request.use((config) => { return config; }); -const pushFormDataValue = (form: FormData, key: string, value: any) => { - if (value?.hasOwnProperty('value') && value.hasOwnProperty('options')) { - form.append(key, value.value, value.options); - } else { - form.append(key, value); - } -}; - -const createFormDataObject = (data: Record) => { - const formData = new FormData(); - const keys = Object.keys(data); - keys.forEach((key) => { - const formField = data[key]; - - if (formField instanceof Array) { - formField.forEach((item) => { - pushFormDataValue(formData, key, item); - }); - } else { - pushFormDataValue(formData, key, formField); - } - }); - return formData; -}; - export const validateUrl = (url?: string): boolean => { if (!url) return false; @@ -183,29 +128,6 @@ function searchForHeader(config: AxiosRequestConfig, headerName: string) { return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName); } -async function generateContentLengthHeader(config: AxiosRequestConfig) { - if (!(config.data instanceof FormData)) { - return; - } - try { - const length = await new Promise((res, rej) => { - config.data.getLength((error: Error | null, length: number) => { - if (error) { - rej(error); - return; - } - res(length); - }); - }); - config.headers = { - ...config.headers, - 'content-length': length, - }; - } catch (error) { - Container.get(Logger).error('Unable to calculate form data length', { error }); - } -} - const getHostFromRequestObject = ( requestObject: Partial<{ url: string; @@ -239,351 +161,6 @@ const getBeforeRedirectFn = } }; -// eslint-disable-next-line complexity -export async function parseRequestObject(requestObject: IRequestOptions) { - // This function is a temporary implementation - // That translates all http requests done via - // the request library to axios directly - // We are not using n8n's interface as it would - // an unnecessary step, considering the `request` - // helper can be deprecated and removed. - const axiosConfig: AxiosRequestConfig = {}; - - if (requestObject.headers !== undefined) { - axiosConfig.headers = requestObject.headers as AxiosHeaders; - } - - // Let's start parsing the hardest part, which is the request body. - // The process here is as following? - // - Check if we have a `content-type` header. If this was set, - // we will follow - // - Check if the `form` property was set. If yes, then it's x-www-form-urlencoded - // - Check if the `formData` property exists. If yes, then it's multipart/form-data - // - Lastly, we should have a regular `body` that is probably a JSON. - - const contentTypeHeaderKeyName = - axiosConfig.headers && - Object.keys(axiosConfig.headers).find( - (headerName) => headerName.toLowerCase() === 'content-type', - ); - const contentType = - contentTypeHeaderKeyName && - (axiosConfig.headers?.[contentTypeHeaderKeyName] as string | undefined); - if (contentType === 'application/x-www-form-urlencoded' && requestObject.formData === undefined) { - // there are nodes incorrectly created, informing the content type header - // and also using formData. Request lib takes precedence for the formData. - // We will do the same. - // Merge body and form properties. - if (typeof requestObject.body === 'string') { - axiosConfig.data = requestObject.body; - } else { - const allData = Object.assign(requestObject.body || {}, requestObject.form || {}) as Record< - string, - string - >; - if (requestObject.useQuerystring === true) { - axiosConfig.data = stringify(allData, { arrayFormat: 'repeat' }); - } else { - axiosConfig.data = stringify(allData); - } - } - } else if (contentType?.includes('multipart/form-data')) { - if (requestObject.formData !== undefined && requestObject.formData instanceof FormData) { - axiosConfig.data = requestObject.formData; - } else { - const allData: Partial = { - ...(requestObject.body as object | undefined), - ...(requestObject.formData as object | undefined), - }; - - axiosConfig.data = createFormDataObject(allData); - } - // replace the existing header with a new one that - // contains the boundary property. - delete axiosConfig.headers?.[contentTypeHeaderKeyName!]; - const headers = axiosConfig.data.getHeaders(); - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); - await generateContentLengthHeader(axiosConfig); - } else { - // When using the `form` property it means the content should be x-www-form-urlencoded. - if (requestObject.form !== undefined && requestObject.body === undefined) { - // If we have only form - axiosConfig.data = - typeof requestObject.form === 'string' - ? stringify(requestObject.form, { format: 'RFC3986' }) - : stringify(requestObject.form).toString(); - if (axiosConfig.headers !== undefined) { - const headerName = searchForHeader(axiosConfig, 'content-type'); - if (headerName) { - delete axiosConfig.headers[headerName]; - } - axiosConfig.headers['Content-Type'] = 'application/x-www-form-urlencoded'; - } else { - axiosConfig.headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - }; - } - } else if (requestObject.formData !== undefined) { - // remove any "content-type" that might exist. - if (axiosConfig.headers !== undefined) { - const headers = Object.keys(axiosConfig.headers); - headers.forEach((header) => { - if (header.toLowerCase() === 'content-type') { - delete axiosConfig.headers?.[header]; - } - }); - } - - if (requestObject.formData instanceof FormData) { - axiosConfig.data = requestObject.formData; - } else { - axiosConfig.data = createFormDataObject(requestObject.formData as Record); - } - // Mix in headers as FormData creates the boundary. - const headers = axiosConfig.data.getHeaders(); - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); - await generateContentLengthHeader(axiosConfig); - } else if (requestObject.body !== undefined) { - // If we have body and possibly form - if (requestObject.form !== undefined && requestObject.body) { - // merge both objects when exist. - requestObject.body = Object.assign(requestObject.body, requestObject.form); - } - axiosConfig.data = requestObject.body as FormData | GenericValue | GenericValue[]; - } - } - - if (requestObject.uri !== undefined) { - axiosConfig.url = requestObject.uri?.toString(); - } - - if (requestObject.url !== undefined) { - axiosConfig.url = requestObject.url?.toString(); - } - - if (requestObject.baseURL !== undefined) { - axiosConfig.baseURL = requestObject.baseURL?.toString(); - } - - if (requestObject.method !== undefined) { - axiosConfig.method = requestObject.method; - } - - if (requestObject.qs !== undefined && Object.keys(requestObject.qs as object).length > 0) { - axiosConfig.params = requestObject.qs; - } - - function hasArrayFormatOptions( - arg: IRequestOptions, - ): arg is Required> { - if ( - typeof arg.qsStringifyOptions === 'object' && - arg.qsStringifyOptions !== null && - !Array.isArray(arg.qsStringifyOptions) && - 'arrayFormat' in arg.qsStringifyOptions - ) { - return true; - } - - return false; - } - - if ( - requestObject.useQuerystring === true || - (hasArrayFormatOptions(requestObject) && - requestObject.qsStringifyOptions.arrayFormat === 'repeat') - ) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'repeat' }); - }; - } else if (requestObject.useQuerystring === false) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'indices' }); - }; - } - - if ( - hasArrayFormatOptions(requestObject) && - requestObject.qsStringifyOptions.arrayFormat === 'brackets' - ) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'brackets' }); - }; - } - - if (requestObject.auth !== undefined) { - // Check support for sendImmediately - if (requestObject.auth.bearer !== undefined) { - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - Authorization: `Bearer ${requestObject.auth.bearer}`, - }); - } else { - const authObj = requestObject.auth; - // Request accepts both user/username and pass/password - axiosConfig.auth = { - username: (authObj.user || authObj.username) as string, - password: (authObj.password || authObj.pass) as string, - }; - } - } - - // Only set header if we have a body, otherwise it may fail - if (requestObject.json === true) { - // Add application/json headers - do not set charset as it breaks a lot of stuff - // only add if no other accept headers was sent. - const acceptHeaderExists = - axiosConfig.headers === undefined - ? false - : Object.keys(axiosConfig.headers) - .map((headerKey) => headerKey.toLowerCase()) - .includes('accept'); - if (!acceptHeaderExists) { - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - Accept: 'application/json', - }); - } - } - if (requestObject.json === false || requestObject.json === undefined) { - // Prevent json parsing - axiosConfig.transformResponse = (res) => res; - } - - // Axios will follow redirects by default, so we simply tell it otherwise if needed. - const { method } = requestObject; - if ( - (requestObject.followRedirect !== false && - (!method || method === 'GET' || method === 'HEAD')) || - requestObject.followAllRedirects - ) { - axiosConfig.maxRedirects = requestObject.maxRedirects; - } else { - axiosConfig.maxRedirects = 0; - } - - const host = getHostFromRequestObject(requestObject); - const agentOptions: AgentOptions = { ...requestObject.agentOptions }; - if (host) { - agentOptions.servername = host; - } - if (requestObject.rejectUnauthorized === false) { - agentOptions.rejectUnauthorized = false; - agentOptions.secureOptions = crypto.constants.SSL_OP_LEGACY_SERVER_CONNECT; - } - - axiosConfig.httpsAgent = new Agent(agentOptions); - - axiosConfig.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosConfig); - - if (requestObject.timeout !== undefined) { - axiosConfig.timeout = requestObject.timeout; - } - - if (requestObject.proxy !== undefined) { - // try our best to parse the url provided. - if (typeof requestObject.proxy === 'string') { - try { - const url = new URL(requestObject.proxy); - const host = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; - axiosConfig.proxy = { - host, - port: parseInt(url.port, 10), - protocol: url.protocol, - }; - if (!url.port) { - // Sets port to a default if not informed - if (url.protocol === 'http') { - axiosConfig.proxy.port = 80; - } else if (url.protocol === 'https') { - axiosConfig.proxy.port = 443; - } - } - if (url.username || url.password) { - axiosConfig.proxy.auth = { - username: url.username, - password: url.password, - }; - } - } catch (error) { - // Not a valid URL. We will try to simply parse stuff - // such as user:pass@host:port without protocol (we'll assume http) - if (requestObject.proxy.includes('@')) { - const [userpass, hostport] = requestObject.proxy.split('@'); - const [username, password] = userpass.split(':'); - const [hostname, port] = hostport.split(':'); - const host = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname; - axiosConfig.proxy = { - host, - port: parseInt(port, 10), - protocol: 'http', - auth: { - username, - password, - }, - }; - } else if (requestObject.proxy.includes(':')) { - const [hostname, port] = requestObject.proxy.split(':'); - axiosConfig.proxy = { - host: hostname, - port: parseInt(port, 10), - protocol: 'http', - }; - } else { - axiosConfig.proxy = { - host: requestObject.proxy, - port: 80, - protocol: 'http', - }; - } - } - } else { - axiosConfig.proxy = requestObject.proxy; - } - } - - if (requestObject.useStream) { - axiosConfig.responseType = 'stream'; - } else if (requestObject.encoding === null) { - // When downloading files, return an arrayBuffer. - axiosConfig.responseType = 'arraybuffer'; - } - - // If we don't set an accept header - // Axios forces "application/json, text/plan, */*" - // Which causes some nodes like NextCloud to break - // as the service returns XML unless requested otherwise. - const allHeaders = axiosConfig.headers ? Object.keys(axiosConfig.headers) : []; - if (!allHeaders.some((headerKey) => headerKey.toLowerCase() === 'accept')) { - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { accept: '*/*' }); - } - if ( - requestObject.json !== false && - axiosConfig.data !== undefined && - axiosConfig.data !== '' && - !(axiosConfig.data instanceof Buffer) && - !allHeaders.some((headerKey) => headerKey.toLowerCase() === 'content-type') - ) { - // Use default header for application/json - // If we don't specify this here, axios will add - // application/json; charset=utf-8 - // and this breaks a lot of stuff - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - 'content-type': 'application/json', - }); - } - - if (requestObject.simple === false) { - axiosConfig.validateStatus = () => true; - } - - /** - * Missing properties: - * encoding (need testing) - * gzip (ignored - default already works) - * resolveWithFullResponse (implemented elsewhere) - */ - return axiosConfig; -} - function digestAuthAxiosConfig( axiosConfig: AxiosRequestConfig, response: AxiosResponse, @@ -635,96 +212,6 @@ function digestAuthAxiosConfig( return axiosConfig; } -interface IContentType { - type: string; - parameters: { - charset: string; - [key: string]: string; - }; -} - -interface IContentDisposition { - type: string; - filename?: string; -} - -function parseHeaderParameters(parameters: string[]): Record { - return parameters.reduce( - (acc, param) => { - const [key, value] = param.split('='); - let decodedValue = decodeURIComponent(value).trim(); - if (decodedValue.startsWith('"') && decodedValue.endsWith('"')) { - decodedValue = decodedValue.slice(1, -1); - } - acc[key.toLowerCase().trim()] = decodedValue; - return acc; - }, - {} as Record, - ); -} - -export function parseContentType(contentType?: string): IContentType | null { - if (!contentType) { - return null; - } - - const [type, ...parameters] = contentType.split(';'); - - return { - type: type.toLowerCase(), - parameters: { charset: 'utf-8', ...parseHeaderParameters(parameters) }, - }; -} - -export function parseContentDisposition(contentDisposition?: string): IContentDisposition | null { - if (!contentDisposition) { - return null; - } - - // This is invalid syntax, but common - // Example 'filename="example.png"' (instead of 'attachment; filename="example.png"') - if (!contentDisposition.startsWith('attachment') && !contentDisposition.startsWith('inline')) { - contentDisposition = `attachment; ${contentDisposition}`; - } - - const [type, ...parameters] = contentDisposition.split(';'); - - const parsedParameters = parseHeaderParameters(parameters); - - let { filename } = parsedParameters; - const wildcard = parsedParameters['filename*']; - if (wildcard) { - // https://datatracker.ietf.org/doc/html/rfc5987 - const [_encoding, _locale, content] = wildcard?.split("'") ?? []; - filename = content; - } - - return { type, filename }; -} - -export function parseIncomingMessage(message: IncomingMessage) { - const contentType = parseContentType(message.headers['content-type']); - if (contentType) { - const { type, parameters } = contentType; - message.contentType = type; - message.encoding = parameters.charset.toLowerCase() as BufferEncoding; - } - - const contentDisposition = parseContentDisposition(message.headers['content-disposition']); - if (contentDisposition) { - message.contentDisposition = contentDisposition; - } -} - -export async function binaryToString(body: Buffer | Readable, encoding?: string) { - if (!encoding && body instanceof IncomingMessage) { - parseIncomingMessage(body); - encoding = body.encoding; - } - const buffer = await binaryToBuffer(body); - return iconv.decode(buffer, encoding ?? 'utf-8'); -} - export async function invokeAxios( axiosConfig: AxiosRequestConfig, authOptions: IRequestOptions['auth'] = {}, @@ -745,6 +232,9 @@ export async function invokeAxios( } } +/** + * @deprecated This is only used by legacy request helpers, that are also deprecated + */ export async function proxyRequestToAxios( workflow: Workflow | undefined, additionalData: IWorkflowExecuteAdditionalData | undefined, @@ -773,7 +263,7 @@ export async function proxyRequestToAxios( } else if (body === '') { body = axiosConfig.responseType === 'arraybuffer' ? Buffer.alloc(0) : undefined; } - await additionalData?.hooks?.executeHookFunctions('nodeFetchedData', [workflow?.id, node]); + await additionalData?.hooks?.runHook('nodeFetchedData', [workflow?.id, node]); return configObject.resolveWithFullResponse ? { body, @@ -980,318 +470,6 @@ export async function httpRequest( return result.data; } -export function getBinaryPath(binaryDataId: string): string { - return Container.get(BinaryDataService).getPath(binaryDataId); -} - -/** - * Returns binary file metadata - */ -export async function getBinaryMetadata(binaryDataId: string): Promise { - return await Container.get(BinaryDataService).getMetadata(binaryDataId); -} - -/** - * Returns binary file stream for piping - */ -export async function getBinaryStream(binaryDataId: string, chunkSize?: number): Promise { - return await Container.get(BinaryDataService).getAsStream(binaryDataId, chunkSize); -} - -export function assertBinaryData( - inputData: ITaskDataConnections, - node: INode, - itemIndex: number, - propertyName: string, - inputIndex: number, -): IBinaryData { - const binaryKeyData = inputData.main[inputIndex]![itemIndex].binary; - if (binaryKeyData === undefined) { - throw new NodeOperationError( - node, - `This operation expects the node's input data to contain a binary file '${propertyName}', but none was found [item ${itemIndex}]`, - { - itemIndex, - description: 'Make sure that the previous node outputs a binary file', - }, - ); - } - - const binaryPropertyData = binaryKeyData[propertyName]; - if (binaryPropertyData === undefined) { - throw new NodeOperationError( - node, - `The item has no binary field '${propertyName}' [item ${itemIndex}]`, - { - itemIndex, - description: - 'Check that the parameter where you specified the input binary field name is correct, and that it matches a field in the binary input', - }, - ); - } - - return binaryPropertyData; -} - -/** - * Returns binary data buffer for given item index and property name. - */ -export async function getBinaryDataBuffer( - inputData: ITaskDataConnections, - itemIndex: number, - propertyName: string, - inputIndex: number, -): Promise { - const binaryData = inputData.main[inputIndex]![itemIndex].binary![propertyName]; - return await Container.get(BinaryDataService).getAsBuffer(binaryData); -} - -export function detectBinaryEncoding(buffer: Buffer): string { - return chardet.detect(buffer) as string; -} - -/** - * Store an incoming IBinaryData & related buffer using the configured binary data manager. - * - * @export - * @param {IBinaryData} binaryData - * @param {Buffer | Readable} bufferOrStream - * @returns {Promise} - */ -export async function setBinaryDataBuffer( - binaryData: IBinaryData, - bufferOrStream: Buffer | Readable, - workflowId: string, - executionId: string, -): Promise { - return await Container.get(BinaryDataService).store( - workflowId, - executionId, - bufferOrStream, - binaryData, - ); -} - -export async function copyBinaryFile( - workflowId: string, - executionId: string, - filePath: string, - fileName: string, - mimeType?: string, -): Promise { - let fileExtension: string | undefined; - if (!mimeType) { - // If no mime type is given figure it out - - if (filePath) { - // Use file path to guess mime type - const mimeTypeLookup = lookup(filePath); - if (mimeTypeLookup) { - mimeType = mimeTypeLookup; - } - } - - if (!mimeType) { - // read the first bytes of the file to guess mime type - const fileTypeData = await FileType.fromFile(filePath); - if (fileTypeData) { - mimeType = fileTypeData.mime; - fileExtension = fileTypeData.ext; - } - } - } - - if (!fileExtension && mimeType) { - fileExtension = extension(mimeType) || undefined; - } - - if (!mimeType) { - // Fall back to text - mimeType = 'text/plain'; - } - - const returnData: IBinaryData = { - mimeType, - fileType: fileTypeFromMimeType(mimeType), - fileExtension, - data: '', - }; - - if (fileName) { - returnData.fileName = fileName; - } else if (filePath) { - returnData.fileName = path.parse(filePath).base; - } - - return await Container.get(BinaryDataService).copyBinaryFile( - workflowId, - executionId, - returnData, - filePath, - ); -} - -/** - * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as - * base64 and adds metadata. - */ -// eslint-disable-next-line complexity -export async function prepareBinaryData( - binaryData: Buffer | Readable, - executionId: string, - workflowId: string, - filePath?: string, - mimeType?: string, -): Promise { - let fileExtension: string | undefined; - if (binaryData instanceof IncomingMessage) { - if (!filePath) { - try { - const { responseUrl } = binaryData; - filePath = - binaryData.contentDisposition?.filename ?? - ((responseUrl && new URL(responseUrl).pathname) ?? binaryData.req?.path)?.slice(1); - } catch {} - } - if (!mimeType) { - mimeType = binaryData.contentType; - } - } - - if (!mimeType) { - // If no mime type is given figure it out - - if (filePath) { - // Use file path to guess mime type - const mimeTypeLookup = lookup(filePath); - if (mimeTypeLookup) { - mimeType = mimeTypeLookup; - } - } - - if (!mimeType) { - if (Buffer.isBuffer(binaryData)) { - // Use buffer to guess mime type - const fileTypeData = await FileType.fromBuffer(binaryData); - if (fileTypeData) { - mimeType = fileTypeData.mime; - fileExtension = fileTypeData.ext; - } - } else if (binaryData instanceof IncomingMessage) { - mimeType = binaryData.headers['content-type']; - } else { - // TODO: detect filetype from other kind of streams - } - } - } - - if (!fileExtension && mimeType) { - fileExtension = extension(mimeType) || undefined; - } - - if (!mimeType) { - // Fall back to text - mimeType = 'text/plain'; - } - - const returnData: IBinaryData = { - mimeType, - fileType: fileTypeFromMimeType(mimeType), - fileExtension, - data: '', - }; - - if (filePath) { - if (filePath.includes('?')) { - // Remove maybe present query parameters - filePath = filePath.split('?').shift(); - } - - const filePathParts = path.parse(filePath as string); - - if (filePathParts.dir !== '') { - returnData.directory = filePathParts.dir; - } - returnData.fileName = filePathParts.base; - - // Remove the dot - const fileExtension = filePathParts.ext.slice(1); - if (fileExtension) { - returnData.fileExtension = fileExtension; - } - } - - return await setBinaryDataBuffer(returnData, binaryData, workflowId, executionId); -} - -export async function checkProcessedAndRecord( - items: DeduplicationItemTypes[], - scope: DeduplicationScope, - contextData: ICheckProcessedContextData, - options: ICheckProcessedOptions, -): Promise { - return await DataDeduplicationService.getInstance().checkProcessedAndRecord( - items, - scope, - contextData, - options, - ); -} - -export async function checkProcessedItemsAndRecord( - key: string, - items: IDataObject[], - scope: DeduplicationScope, - contextData: ICheckProcessedContextData, - options: ICheckProcessedOptions, -): Promise { - return await DataDeduplicationService.getInstance().checkProcessedItemsAndRecord( - key, - items, - scope, - contextData, - options, - ); -} - -export async function removeProcessed( - items: DeduplicationItemTypes[], - scope: DeduplicationScope, - contextData: ICheckProcessedContextData, - options: ICheckProcessedOptions, -): Promise { - return await DataDeduplicationService.getInstance().removeProcessed( - items, - scope, - contextData, - options, - ); -} - -export async function clearAllProcessedItems( - scope: DeduplicationScope, - contextData: ICheckProcessedContextData, - options: ICheckProcessedOptions, -): Promise { - return await DataDeduplicationService.getInstance().clearAllProcessedItems( - scope, - contextData, - options, - ); -} - -export async function getProcessedDataCount( - scope: DeduplicationScope, - contextData: ICheckProcessedContextData, - options: ICheckProcessedOptions, -): Promise { - return await DataDeduplicationService.getInstance().getProcessedDataCount( - scope, - contextData, - options, - ); -} - export function applyPaginationRequestData( requestData: IRequestOptions, paginationRequestData: PaginationOptions['request'], @@ -2341,180 +1519,6 @@ export const getSchedulingFunctions = (workflow: Workflow): SchedulingFunctions }; }; -const getAllowedPaths = () => { - const restrictFileAccessTo = process.env[RESTRICT_FILE_ACCESS_TO]; - if (!restrictFileAccessTo) { - return []; - } - const allowedPaths = restrictFileAccessTo - .split(';') - .map((path) => path.trim()) - .filter((path) => path); - return allowedPaths; -}; - -export function isFilePathBlocked(filePath: string): boolean { - const allowedPaths = getAllowedPaths(); - const resolvedFilePath = path.resolve(filePath); - const blockFileAccessToN8nFiles = process.env[BLOCK_FILE_ACCESS_TO_N8N_FILES] !== 'false'; - - //if allowed paths are defined, allow access only to those paths - if (allowedPaths.length) { - for (const path of allowedPaths) { - if (resolvedFilePath.startsWith(path)) { - return false; - } - } - - return true; - } - - //restrict access to .n8n folder, ~/.cache/n8n/public, and other .env config related paths - if (blockFileAccessToN8nFiles) { - const { n8nFolder, staticCacheDir } = Container.get(InstanceSettings); - const restrictedPaths = [n8nFolder, staticCacheDir]; - - if (process.env[CONFIG_FILES]) { - restrictedPaths.push(...process.env[CONFIG_FILES].split(',')); - } - - if (process.env[CUSTOM_EXTENSION_ENV]) { - const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV].split(';'); - restrictedPaths.push(...customExtensionFolders); - } - - if (process.env[BINARY_DATA_STORAGE_PATH]) { - restrictedPaths.push(process.env[BINARY_DATA_STORAGE_PATH]); - } - - if (process.env[UM_EMAIL_TEMPLATES_INVITE]) { - restrictedPaths.push(process.env[UM_EMAIL_TEMPLATES_INVITE]); - } - - if (process.env[UM_EMAIL_TEMPLATES_PWRESET]) { - restrictedPaths.push(process.env[UM_EMAIL_TEMPLATES_PWRESET]); - } - - //check if the file path is restricted - for (const path of restrictedPaths) { - if (resolvedFilePath.startsWith(path)) { - return true; - } - } - } - - //path is not restricted - return false; -} - -export const getFileSystemHelperFunctions = (node: INode): FileSystemHelperFunctions => ({ - async createReadStream(filePath) { - try { - await fsAccess(filePath); - } catch (error) { - throw error.code === 'ENOENT' - ? new NodeOperationError(node, error, { - message: `The file "${String(filePath)}" could not be accessed.`, - level: 'warning', - }) - : error; - } - if (isFilePathBlocked(filePath as string)) { - const allowedPaths = getAllowedPaths(); - const message = allowedPaths.length ? ` Allowed paths: ${allowedPaths.join(', ')}` : ''; - throw new NodeOperationError(node, `Access to the file is not allowed.${message}`, { - level: 'warning', - }); - } - return createReadStream(filePath); - }, - - getStoragePath() { - return path.join(Container.get(InstanceSettings).n8nFolder, `storage/${node.type}`); - }, - - async writeContentToFile(filePath, content, flag) { - if (isFilePathBlocked(filePath as string)) { - throw new NodeOperationError(node, `The file "${String(filePath)}" is not writable.`, { - level: 'warning', - }); - } - return await fsWriteFile(filePath, content, { encoding: 'binary', flag }); - }, -}); - -export const getNodeHelperFunctions = ( - { executionId }: IWorkflowExecuteAdditionalData, - workflowId: string, -): NodeHelperFunctions => ({ - copyBinaryFile: async (filePath, fileName, mimeType) => - await copyBinaryFile(workflowId, executionId!, filePath, fileName, mimeType), -}); - -export const getBinaryHelperFunctions = ( - { executionId }: IWorkflowExecuteAdditionalData, - workflowId: string, -): BinaryHelperFunctions => ({ - getBinaryPath, - getBinaryStream, - getBinaryMetadata, - binaryToBuffer, - binaryToString, - prepareBinaryData: async (binaryData, filePath, mimeType) => - await prepareBinaryData(binaryData, executionId!, workflowId, filePath, mimeType), - setBinaryDataBuffer: async (data, binaryData) => - await setBinaryDataBuffer(data, binaryData, workflowId, executionId!), - copyBinaryFile: async () => { - throw new ApplicationError('`copyBinaryFile` has been removed. Please upgrade this node.'); - }, -}); - -export const getCheckProcessedHelperFunctions = ( - workflow: Workflow, - node: INode, -): DeduplicationHelperFunctions => ({ - async checkProcessedAndRecord( - items: DeduplicationItemTypes[], - scope: DeduplicationScope, - options: ICheckProcessedOptions, - ): Promise { - return await checkProcessedAndRecord(items, scope, { node, workflow }, options); - }, - async checkProcessedItemsAndRecord( - propertyName: string, - items: IDataObject[], - scope: DeduplicationScope, - options: ICheckProcessedOptions, - ): Promise { - return await checkProcessedItemsAndRecord( - propertyName, - items, - scope, - { node, workflow }, - options, - ); - }, - async removeProcessed( - items: DeduplicationItemTypes[], - scope: DeduplicationScope, - options: ICheckProcessedOptions, - ): Promise { - return await removeProcessed(items, scope, { node, workflow }, options); - }, - async clearAllProcessedItems( - scope: DeduplicationScope, - options: ICheckProcessedOptions, - ): Promise { - return await clearAllProcessedItems(scope, { node, workflow }, options); - }, - async getProcessedDataCount( - scope: DeduplicationScope, - options: ICheckProcessedOptions, - ): Promise { - return await getProcessedDataCount(scope, { node, workflow }, options); - }, -}); - /** * Returns a copy of the items which only contains the json data and * of that only the defined properties @@ -2560,15 +1564,3 @@ export function getExecuteTriggerFunctions( ): ITriggerFunctions { return new TriggerContext(workflow, node, additionalData, mode, activation); } - -export function getCredentialTestFunctions(): ICredentialTestFunctions { - return { - logger: Container.get(Logger), - helpers: { - ...getSSHTunnelFunctions(), - request: async (uriOrObject: string | object, options?: object) => { - return await proxyRequestToAxios(undefined, undefined, undefined, uriOrObject, options); - }, - }, - }; -} diff --git a/packages/core/src/node-execution-context/helpers/binary-helpers.ts b/packages/core/src/node-execution-context/helpers/binary-helpers.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/core/test/DirectoryLoader.test.ts b/packages/core/src/nodes-loader/__tests__/directory-loader.test.ts similarity index 95% rename from packages/core/test/DirectoryLoader.test.ts rename to packages/core/src/nodes-loader/__tests__/directory-loader.test.ts index 01a8c8d34a..cb66300257 100644 --- a/packages/core/test/DirectoryLoader.test.ts +++ b/packages/core/src/nodes-loader/__tests__/directory-loader.test.ts @@ -22,12 +22,10 @@ jest.mock('fast-glob', () => async (pattern: string) => { : ['dist/Credential1.js']; }); -import * as classLoader from '@/ClassLoader'; -import { - CustomDirectoryLoader, - PackageDirectoryLoader, - LazyPackageDirectoryLoader, -} from '@/DirectoryLoader'; +import { CustomDirectoryLoader } from '../custom-directory-loader'; +import { LazyPackageDirectoryLoader } from '../lazy-package-directory-loader'; +import * as classLoader from '../load-class-in-isolation'; +import { PackageDirectoryLoader } from '../package-directory-loader'; describe('DirectoryLoader', () => { const directory = '/not/a/real/path'; @@ -235,10 +233,7 @@ describe('DirectoryLoader', () => { return JSON.stringify({}); } if (path.endsWith('types/nodes.json')) { - return JSON.stringify([ - { name: 'n8n-nodes-testing.node1' }, - { name: 'n8n-nodes-testing.node2' }, - ]); + return JSON.stringify([{ name: 'node1' }, { name: 'node2' }]); } if (path.endsWith('types/credentials.json')) { return JSON.stringify([]); @@ -254,7 +249,7 @@ describe('DirectoryLoader', () => { node1: { className: 'Node1', sourcePath: 'dist/Node1/Node1.node.js' }, }); expect(loader.types.nodes).toHaveLength(1); - expect(loader.types.nodes[0].name).toBe('n8n-nodes-testing.node1'); + expect(loader.types.nodes[0].name).toBe('node1'); expect(classLoader.loadClassInIsolation).not.toHaveBeenCalled(); }); @@ -274,10 +269,7 @@ describe('DirectoryLoader', () => { return JSON.stringify({}); } if (path.endsWith('types/nodes.json')) { - return JSON.stringify([ - { name: 'n8n-nodes-testing.node1' }, - { name: 'n8n-nodes-testing.node2' }, - ]); + return JSON.stringify([{ name: 'node1' }, { name: 'node2' }]); } if (path.endsWith('types/credentials.json')) { return JSON.stringify([]); @@ -314,10 +306,7 @@ describe('DirectoryLoader', () => { return JSON.stringify({}); } if (path.endsWith('types/nodes.json')) { - return JSON.stringify([ - { name: 'n8n-nodes-testing.node1' }, - { name: 'n8n-nodes-testing.node2' }, - ]); + return JSON.stringify([{ name: 'node1' }, { name: 'node2' }]); } if (path.endsWith('types/credentials.json')) { return JSON.stringify([]); @@ -333,7 +322,7 @@ describe('DirectoryLoader', () => { node2: { className: 'Node2', sourcePath: 'dist/Node2/Node2.node.js' }, }); expect(loader.types.nodes).toHaveLength(1); - expect(loader.types.nodes[0].name).toBe('n8n-nodes-testing.node2'); + expect(loader.types.nodes[0].name).toBe('node2'); expect(classLoader.loadClassInIsolation).not.toHaveBeenCalled(); }); }); @@ -654,18 +643,6 @@ describe('DirectoryLoader', () => { expect(nodeWithIcon.description.icon).toBeUndefined(); }); - it('should skip node if included in excludeNodes', () => { - const loader = new CustomDirectoryLoader(directory, ['CUSTOM.node1']); - const filePath = 'dist/Node1/Node1.node.js'; - - loader.loadNodeFromFile(filePath); - - expect(loader.nodeTypes).toEqual({}); - expect(loader.known.nodes).toEqual({}); - expect(loader.types.nodes).toEqual([]); - expect(loader.loadedNodes).toEqual([]); - }); - it('should skip node if not in includeNodes', () => { const loader = new CustomDirectoryLoader(directory, [], ['CUSTOM.other']); const filePath = 'dist/Node1/Node1.node.js'; diff --git a/packages/core/test/ClassLoader.test.ts b/packages/core/src/nodes-loader/__tests__/load-class-in-isolation.test.ts similarity index 93% rename from packages/core/test/ClassLoader.test.ts rename to packages/core/src/nodes-loader/__tests__/load-class-in-isolation.test.ts index 9527572662..656830fa87 100644 --- a/packages/core/test/ClassLoader.test.ts +++ b/packages/core/src/nodes-loader/__tests__/load-class-in-isolation.test.ts @@ -1,8 +1,8 @@ import vm from 'vm'; -import { loadClassInIsolation } from '@/ClassLoader'; +import { loadClassInIsolation } from '../load-class-in-isolation'; -describe('ClassLoader', () => { +describe('loadClassInIsolation', () => { const filePath = '/path/to/TestClass.js'; const className = 'TestClass'; diff --git a/packages/core/src/nodes-loader/constants.ts b/packages/core/src/nodes-loader/constants.ts new file mode 100644 index 0000000000..171be0352b --- /dev/null +++ b/packages/core/src/nodes-loader/constants.ts @@ -0,0 +1,31 @@ +import type { INodeProperties } from 'n8n-workflow'; +import { cronNodeOptions } from 'n8n-workflow'; + +export const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; + +export const commonPollingParameters: INodeProperties[] = [ + { + displayName: 'Poll Times', + name: 'pollTimes', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + multipleValueButtonText: 'Add Poll Time', + }, + default: { item: [{ mode: 'everyMinute' }] }, + description: 'Time at which polling should occur', + placeholder: 'Add Poll Time', + options: cronNodeOptions, + }, +]; + +export const commonCORSParameters: INodeProperties[] = [ + { + displayName: 'Allowed Origins (CORS)', + name: 'allowedOrigins', + type: 'string', + default: '*', + description: + 'Comma-separated list of URLs allowed for cross-origin non-preflight requests. Use * (default) to allow all origins.', + }, +]; diff --git a/packages/core/src/nodes-loader/custom-directory-loader.ts b/packages/core/src/nodes-loader/custom-directory-loader.ts new file mode 100644 index 0000000000..8e84440608 --- /dev/null +++ b/packages/core/src/nodes-loader/custom-directory-loader.ts @@ -0,0 +1,31 @@ +import glob from 'fast-glob'; + +import { DirectoryLoader } from './directory-loader'; + +/** + * Loader for source files of nodes and credentials located in a custom dir, + * e.g. `~/.n8n/custom` + */ +export class CustomDirectoryLoader extends DirectoryLoader { + packageName = 'CUSTOM'; + + override async loadAll() { + const nodes = await glob('**/*.node.js', { + cwd: this.directory, + absolute: true, + }); + + for (const nodePath of nodes) { + this.loadNodeFromFile(nodePath); + } + + const credentials = await glob('**/*.credentials.js', { + cwd: this.directory, + absolute: true, + }); + + for (const credentialPath of credentials) { + this.loadCredentialFromFile(credentialPath); + } + } +} diff --git a/packages/core/src/DirectoryLoader.ts b/packages/core/src/nodes-loader/directory-loader.ts similarity index 66% rename from packages/core/src/DirectoryLoader.ts rename to packages/core/src/nodes-loader/directory-loader.ts index 26dace0fd1..094aa8d1e0 100644 --- a/packages/core/src/DirectoryLoader.ts +++ b/packages/core/src/nodes-loader/directory-loader.ts @@ -1,5 +1,4 @@ import { Container } from '@n8n/di'; -import glob from 'fast-glob'; import uniqBy from 'lodash/uniqBy'; import type { CodexData, @@ -16,18 +15,15 @@ import type { IVersionedNodeType, KnownNodesAndCredentials, } from 'n8n-workflow'; -import { ApplicationError, applyDeclarativeNodeOptionParameters, jsonParse } from 'n8n-workflow'; -import { readFileSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; +import { ApplicationError, applyDeclarativeNodeOptionParameters } from 'n8n-workflow'; import * as path from 'path'; +import { UnrecognizedCredentialTypeError } from '@/errors/unrecognized-credential-type.error'; +import { UnrecognizedNodeTypeError } from '@/errors/unrecognized-node-type.error'; import { Logger } from '@/logging/logger'; -import { loadClassInIsolation } from './ClassLoader'; -import { commonCORSParameters, commonPollingParameters, CUSTOM_NODES_CATEGORY } from './Constants'; -import { UnrecognizedCredentialTypeError } from './errors/unrecognized-credential-type.error'; -import { UnrecognizedNodeTypeError } from './errors/unrecognized-node-type.error'; -import type { n8n } from './Interfaces'; +import { commonCORSParameters, commonPollingParameters, CUSTOM_NODES_CATEGORY } from './constants'; +import { loadClassInIsolation } from './load-class-in-isolation'; function toJSON(this: ICredentialType) { return { @@ -80,8 +76,8 @@ export abstract class DirectoryLoader { constructor( readonly directory: string, - protected readonly excludeNodes: string[] = [], - protected readonly includeNodes: string[] = [], + protected excludeNodes: string[] = [], + protected includeNodes: string[] = [], ) {} abstract packageName: string; @@ -121,13 +117,12 @@ export abstract class DirectoryLoader { this.addCodex(tempNode, filePath); const nodeType = tempNode.description.name; - const fullNodeType = `${this.packageName}.${nodeType}`; - if (this.includeNodes.length && !this.includeNodes.includes(fullNodeType)) { + if (this.includeNodes.length && !this.includeNodes.includes(nodeType)) { return; } - if (this.excludeNodes.includes(fullNodeType)) { + if (this.excludeNodes.includes(nodeType)) { return; } @@ -151,7 +146,7 @@ export abstract class DirectoryLoader { if (currentVersionNode.hasOwnProperty('executeSingle')) { throw new ApplicationError( '"executeSingle" has been removed. Please update the code of this node to use "execute" instead.', - { extra: { nodeType: fullNodeType } }, + { extra: { nodeType } }, ); } } else { @@ -396,171 +391,3 @@ export abstract class DirectoryLoader { } } } - -/** - * Loader for source files of nodes and credentials located in a custom dir, - * e.g. `~/.n8n/custom` - */ -export class CustomDirectoryLoader extends DirectoryLoader { - packageName = 'CUSTOM'; - - override async loadAll() { - const nodes = await glob('**/*.node.js', { - cwd: this.directory, - absolute: true, - }); - - for (const nodePath of nodes) { - this.loadNodeFromFile(nodePath); - } - - const credentials = await glob('**/*.credentials.js', { - cwd: this.directory, - absolute: true, - }); - - for (const credentialPath of credentials) { - this.loadCredentialFromFile(credentialPath); - } - } -} - -/** - * Loader for source files of nodes and credentials located in a package dir, - * e.g. /nodes-base or community packages. - */ -export class PackageDirectoryLoader extends DirectoryLoader { - packageJson: n8n.PackageJson = this.readJSONSync('package.json'); - - packageName = this.packageJson.name; - - override async loadAll() { - const { n8n } = this.packageJson; - if (!n8n) return; - - const { nodes, credentials } = n8n; - - if (Array.isArray(nodes)) { - for (const nodePath of nodes) { - this.loadNodeFromFile(nodePath); - } - } - - if (Array.isArray(credentials)) { - for (const credentialPath of credentials) { - this.loadCredentialFromFile(credentialPath); - } - } - - this.inferSupportedNodes(); - - this.logger.debug(`Loaded all credentials and nodes from ${this.packageName}`, { - credentials: credentials?.length ?? 0, - nodes: nodes?.length ?? 0, - }); - } - - private inferSupportedNodes() { - const knownCredentials = this.known.credentials; - for (const { type: credentialType } of Object.values(this.credentialTypes)) { - const supportedNodes = knownCredentials[credentialType.name].supportedNodes ?? []; - if (supportedNodes.length > 0 && credentialType.httpRequestNode) { - credentialType.httpRequestNode.hidden = true; - } - - credentialType.supportedNodes = supportedNodes; - - if (!credentialType.iconUrl && !credentialType.icon) { - for (const supportedNode of supportedNodes) { - const nodeDescription = this.nodeTypes[supportedNode]?.type.description; - - if (!nodeDescription) continue; - if (nodeDescription.icon) { - credentialType.icon = nodeDescription.icon; - credentialType.iconColor = nodeDescription.iconColor; - break; - } - if (nodeDescription.iconUrl) { - credentialType.iconUrl = nodeDescription.iconUrl; - break; - } - } - } - } - } - - private parseJSON(fileString: string, filePath: string): T { - try { - return jsonParse(fileString); - } catch (error) { - throw new ApplicationError('Failed to parse JSON', { extra: { filePath } }); - } - } - - protected readJSONSync(file: string): T { - const filePath = this.resolvePath(file); - const fileString = readFileSync(filePath, 'utf8'); - return this.parseJSON(fileString, filePath); - } - - protected async readJSON(file: string): Promise { - const filePath = this.resolvePath(file); - const fileString = await readFile(filePath, 'utf8'); - return this.parseJSON(fileString, filePath); - } -} - -/** - * This loader extends PackageDirectoryLoader to load node and credentials lazily, if possible - */ -export class LazyPackageDirectoryLoader extends PackageDirectoryLoader { - override async loadAll() { - try { - this.known.nodes = await this.readJSON('dist/known/nodes.json'); - this.known.credentials = await this.readJSON('dist/known/credentials.json'); - - this.types.nodes = await this.readJSON('dist/types/nodes.json'); - this.types.credentials = await this.readJSON('dist/types/credentials.json'); - - if (this.includeNodes.length) { - const allowedNodes: typeof this.known.nodes = {}; - for (const fullNodeType of this.includeNodes) { - const [packageName, nodeType] = fullNodeType.split('.'); - if (packageName === this.packageName && nodeType in this.known.nodes) { - allowedNodes[nodeType] = this.known.nodes[nodeType]; - } - } - this.known.nodes = allowedNodes; - - this.types.nodes = this.types.nodes.filter((nodeType) => - this.includeNodes.includes(nodeType.name), - ); - } - - if (this.excludeNodes.length) { - for (const fullNodeType of this.excludeNodes) { - const [packageName, nodeType] = fullNodeType.split('.'); - if (packageName === this.packageName) { - delete this.known.nodes[nodeType]; - } - } - - this.types.nodes = this.types.nodes.filter( - (nodeType) => !this.excludeNodes.includes(nodeType.name), - ); - } - - this.logger.debug(`Lazy-loading nodes and credentials from ${this.packageJson.name}`, { - nodes: this.types.nodes?.length ?? 0, - credentials: this.types.credentials?.length ?? 0, - }); - - this.isLazyLoaded = true; - - return; // We can load nodes and credentials lazily now - } catch { - this.logger.debug("Can't enable lazy-loading"); - await super.loadAll(); - } - } -} diff --git a/packages/core/src/nodes-loader/index.ts b/packages/core/src/nodes-loader/index.ts new file mode 100644 index 0000000000..f8fedca5be --- /dev/null +++ b/packages/core/src/nodes-loader/index.ts @@ -0,0 +1,5 @@ +export { DirectoryLoader, type Types } from './directory-loader'; +export { CustomDirectoryLoader } from './custom-directory-loader'; +export { PackageDirectoryLoader } from './package-directory-loader'; +export { LazyPackageDirectoryLoader } from './lazy-package-directory-loader'; +export type { n8n } from './types'; diff --git a/packages/core/src/nodes-loader/lazy-package-directory-loader.ts b/packages/core/src/nodes-loader/lazy-package-directory-loader.ts new file mode 100644 index 0000000000..5da6084e1a --- /dev/null +++ b/packages/core/src/nodes-loader/lazy-package-directory-loader.ts @@ -0,0 +1,52 @@ +import { PackageDirectoryLoader } from './package-directory-loader'; + +/** + * This loader extends PackageDirectoryLoader to load node and credentials lazily, if possible + */ +export class LazyPackageDirectoryLoader extends PackageDirectoryLoader { + override async loadAll() { + try { + this.known.nodes = await this.readJSON('dist/known/nodes.json'); + this.known.credentials = await this.readJSON('dist/known/credentials.json'); + + this.types.nodes = await this.readJSON('dist/types/nodes.json'); + this.types.credentials = await this.readJSON('dist/types/credentials.json'); + + if (this.includeNodes.length) { + const allowedNodes: typeof this.known.nodes = {}; + for (const nodeType of this.includeNodes) { + if (nodeType in this.known.nodes) { + allowedNodes[nodeType] = this.known.nodes[nodeType]; + } + } + this.known.nodes = allowedNodes; + + this.types.nodes = this.types.nodes.filter((nodeType) => + this.includeNodes.includes(nodeType.name), + ); + } + + if (this.excludeNodes.length) { + for (const nodeType of this.excludeNodes) { + delete this.known.nodes[nodeType]; + } + + this.types.nodes = this.types.nodes.filter( + (nodeType) => !this.excludeNodes.includes(nodeType.name), + ); + } + + this.logger.debug(`Lazy-loading nodes and credentials from ${this.packageJson.name}`, { + nodes: this.types.nodes?.length ?? 0, + credentials: this.types.credentials?.length ?? 0, + }); + + this.isLazyLoaded = true; + + return; // We can load nodes and credentials lazily now + } catch { + this.logger.debug("Can't enable lazy-loading"); + await super.loadAll(); + } + } +} diff --git a/packages/core/src/ClassLoader.ts b/packages/core/src/nodes-loader/load-class-in-isolation.ts similarity index 100% rename from packages/core/src/ClassLoader.ts rename to packages/core/src/nodes-loader/load-class-in-isolation.ts diff --git a/packages/core/src/nodes-loader/package-directory-loader.ts b/packages/core/src/nodes-loader/package-directory-loader.ts new file mode 100644 index 0000000000..4c46a684ec --- /dev/null +++ b/packages/core/src/nodes-loader/package-directory-loader.ts @@ -0,0 +1,107 @@ +import { ApplicationError, jsonParse } from 'n8n-workflow'; +import { readFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; + +import { DirectoryLoader } from './directory-loader'; +import type { n8n } from './types'; + +/** + * Loader for source files of nodes and credentials located in a package dir, + * e.g. /nodes-base or community packages. + */ +export class PackageDirectoryLoader extends DirectoryLoader { + packageJson: n8n.PackageJson; + + packageName: string; + + constructor(directory: string, excludeNodes: string[] = [], includeNodes: string[] = []) { + super(directory, excludeNodes, includeNodes); + + this.packageJson = this.readJSONSync('package.json'); + this.packageName = this.packageJson.name; + this.excludeNodes = this.extractNodeTypes(excludeNodes); + this.includeNodes = this.extractNodeTypes(includeNodes); + } + + private extractNodeTypes(fullNodeTypes: string[]) { + return fullNodeTypes + .map((fullNodeType) => fullNodeType.split('.')) + .filter(([packageName]) => packageName === this.packageName) + .map(([_, nodeType]) => nodeType); + } + + override async loadAll() { + const { n8n } = this.packageJson; + if (!n8n) return; + + const { nodes, credentials } = n8n; + + if (Array.isArray(nodes)) { + for (const nodePath of nodes) { + this.loadNodeFromFile(nodePath); + } + } + + if (Array.isArray(credentials)) { + for (const credentialPath of credentials) { + this.loadCredentialFromFile(credentialPath); + } + } + + this.inferSupportedNodes(); + + this.logger.debug(`Loaded all credentials and nodes from ${this.packageName}`, { + credentials: credentials?.length ?? 0, + nodes: nodes?.length ?? 0, + }); + } + + private inferSupportedNodes() { + const knownCredentials = this.known.credentials; + for (const { type: credentialType } of Object.values(this.credentialTypes)) { + const supportedNodes = knownCredentials[credentialType.name].supportedNodes ?? []; + if (supportedNodes.length > 0 && credentialType.httpRequestNode) { + credentialType.httpRequestNode.hidden = true; + } + + credentialType.supportedNodes = supportedNodes; + + if (!credentialType.iconUrl && !credentialType.icon) { + for (const supportedNode of supportedNodes) { + const nodeDescription = this.nodeTypes[supportedNode]?.type.description; + + if (!nodeDescription) continue; + if (nodeDescription.icon) { + credentialType.icon = nodeDescription.icon; + credentialType.iconColor = nodeDescription.iconColor; + break; + } + if (nodeDescription.iconUrl) { + credentialType.iconUrl = nodeDescription.iconUrl; + break; + } + } + } + } + } + + private parseJSON(fileString: string, filePath: string): T { + try { + return jsonParse(fileString); + } catch (error) { + throw new ApplicationError('Failed to parse JSON', { extra: { filePath } }); + } + } + + protected readJSONSync(file: string): T { + const filePath = this.resolvePath(file); + const fileString = readFileSync(filePath, 'utf8'); + return this.parseJSON(fileString, filePath); + } + + protected async readJSON(file: string): Promise { + const filePath = this.resolvePath(file); + const fileString = await readFile(filePath, 'utf8'); + return this.parseJSON(fileString, filePath); + } +} diff --git a/packages/core/src/nodes-loader/types.ts b/packages/core/src/nodes-loader/types.ts new file mode 100644 index 0000000000..e898c44b2d --- /dev/null +++ b/packages/core/src/nodes-loader/types.ts @@ -0,0 +1,14 @@ +export namespace n8n { + export interface PackageJson { + name: string; + version: string; + n8n?: { + credentials?: string[]; + nodes?: string[]; + }; + author?: { + name?: string; + email?: string; + }; + } +} diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/utils/__tests__/is-object-literal.test.ts similarity index 95% rename from packages/core/src/__tests__/utils.test.ts rename to packages/core/src/utils/__tests__/is-object-literal.test.ts index a8532ed589..f6c3cfddd0 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/utils/__tests__/is-object-literal.test.ts @@ -1,4 +1,4 @@ -import { isObjectLiteral } from '@/utils'; +import { isObjectLiteral } from '../is-object-literal'; describe('isObjectLiteral', () => { test.each([ diff --git a/packages/core/test/SerializedBuffer.test.ts b/packages/core/src/utils/__tests__/serialized-buffer.test.ts similarity index 92% rename from packages/core/test/SerializedBuffer.test.ts rename to packages/core/src/utils/__tests__/serialized-buffer.test.ts index 95d7213401..19b7df496d 100644 --- a/packages/core/test/SerializedBuffer.test.ts +++ b/packages/core/src/utils/__tests__/serialized-buffer.test.ts @@ -1,5 +1,5 @@ -import type { SerializedBuffer } from '@/SerializedBuffer'; -import { toBuffer, isSerializedBuffer } from '@/SerializedBuffer'; +import type { SerializedBuffer } from '../serialized-buffer'; +import { toBuffer, isSerializedBuffer } from '../serialized-buffer'; // Mock data for tests const validSerializedBuffer: SerializedBuffer = { diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts new file mode 100644 index 0000000000..4973c13417 --- /dev/null +++ b/packages/core/src/utils/index.ts @@ -0,0 +1,2 @@ +export * from './serialized-buffer'; +export { isObjectLiteral } from './is-object-literal'; diff --git a/packages/core/src/utils.ts b/packages/core/src/utils/is-object-literal.ts similarity index 100% rename from packages/core/src/utils.ts rename to packages/core/src/utils/is-object-literal.ts diff --git a/packages/core/src/SerializedBuffer.ts b/packages/core/src/utils/serialized-buffer.ts similarity index 91% rename from packages/core/src/SerializedBuffer.ts rename to packages/core/src/utils/serialized-buffer.ts index d6ea874c7a..e98e1a0883 100644 --- a/packages/core/src/SerializedBuffer.ts +++ b/packages/core/src/utils/serialized-buffer.ts @@ -1,4 +1,4 @@ -import { isObjectLiteral } from '@/utils'; +import { isObjectLiteral } from './is-object-literal'; /** A nodejs Buffer gone through JSON.stringify */ export type SerializedBuffer = { diff --git a/packages/core/test/Credentials.test.ts b/packages/core/test/Credentials.test.ts deleted file mode 100644 index bf6ba29ca5..0000000000 --- a/packages/core/test/Credentials.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Container } from '@n8n/di'; -import { mock } from 'jest-mock-extended'; -import type { CredentialInformation } from 'n8n-workflow'; - -import { Cipher } from '@/Cipher'; -import { Credentials } from '@/Credentials'; -import type { InstanceSettings } from '@/InstanceSettings'; - -describe('Credentials', () => { - const cipher = new Cipher(mock({ encryptionKey: 'password' })); - Container.set(Cipher, cipher); - - const setDataKey = (credentials: Credentials, key: string, data: CredentialInformation) => { - let fullData; - try { - fullData = credentials.getData(); - } catch (e) { - fullData = {}; - } - fullData[key] = data; - return credentials.setData(fullData); - }; - - describe('without nodeType set', () => { - test('should be able to set and read key data without initial data set', () => { - const credentials = new Credentials({ id: null, name: 'testName' }, 'testType'); - - const key = 'key1'; - const newData = 1234; - - setDataKey(credentials, key, newData); - - expect(credentials.getData()[key]).toEqual(newData); - }); - - test('should be able to set and read key data with initial data set', () => { - const key = 'key2'; - - // Saved under "key1" - const initialData = 4321; - const initialDataEncoded = 'U2FsdGVkX1+0baznXt+Ag/ub8A2kHLyoLxn/rR9h4XQ='; - - const credentials = new Credentials( - { id: null, name: 'testName' }, - 'testType', - initialDataEncoded, - ); - - const newData = 1234; - - // Set and read new data - setDataKey(credentials, key, newData); - expect(credentials.getData()[key]).toEqual(newData); - - // Read the data which got provided encrypted on init - expect(credentials.getData().key1).toEqual(initialData); - }); - }); -}); diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts deleted file mode 100644 index 703e3f9bef..0000000000 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ /dev/null @@ -1,932 +0,0 @@ -import { Container } from '@n8n/di'; -import FormData from 'form-data'; -import { mkdtempSync, readFileSync } from 'fs'; -import { IncomingMessage } from 'http'; -import type { Agent } from 'https'; -import { mock } from 'jest-mock-extended'; -import type { - IBinaryData, - IHttpRequestMethods, - IHttpRequestOptions, - INode, - IRequestOptions, - ITaskDataConnections, - IWorkflowExecuteAdditionalData, - Workflow, - WorkflowHooks, -} from 'n8n-workflow'; -import nock from 'nock'; -import { tmpdir } from 'os'; -import { join } from 'path'; -import { Readable } from 'stream'; -import type { SecureContextOptions } from 'tls'; - -import { BinaryDataService } from '@/BinaryData/BinaryData.service'; -import { InstanceSettings } from '@/InstanceSettings'; -import { - binaryToString, - copyInputItems, - getBinaryDataBuffer, - invokeAxios, - isFilePathBlocked, - parseContentDisposition, - parseContentType, - parseIncomingMessage, - parseRequestObject, - proxyRequestToAxios, - removeEmptyBody, - setBinaryDataBuffer, -} from '@/NodeExecuteFunctions'; - -const temporaryDir = mkdtempSync(join(tmpdir(), 'n8n')); - -describe('NodeExecuteFunctions', () => { - describe('test binary data helper methods', () => { - test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => { - // Setup a 'default' binary data manager instance - Container.set(BinaryDataService, new BinaryDataService()); - - await Container.get(BinaryDataService).init({ - mode: 'default', - availableModes: ['default'], - localStoragePath: temporaryDir, - }); - - // Set our binary data buffer - const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); - const setBinaryDataBufferResponse: IBinaryData = await setBinaryDataBuffer( - { - mimeType: 'txt', - data: 'This should be overwritten by the actual payload in the response', - }, - inputData, - 'workflowId', - 'executionId', - ); - - // Expect our return object to contain the base64 encoding of the input data, as it should be stored in memory. - expect(setBinaryDataBufferResponse.data).toEqual(inputData.toString('base64')); - - // Now, re-fetch our data. - // An ITaskDataConnections object is used to share data between nodes. The top level property, 'main', represents the successful output object from a previous node. - const taskDataConnectionsInput: ITaskDataConnections = { - main: [], - }; - - // We add an input set, with one item at index 0, to this input. It contains an empty json payload and our binary data. - taskDataConnectionsInput.main.push([ - { - json: {}, - binary: { - data: setBinaryDataBufferResponse, - }, - }, - ]); - - // Now, lets fetch our data! The item will be item index 0. - const getBinaryDataBufferResponse: Buffer = await getBinaryDataBuffer( - taskDataConnectionsInput, - 0, - 'data', - 0, - ); - - expect(getBinaryDataBufferResponse).toEqual(inputData); - }); - - test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => { - Container.set(BinaryDataService, new BinaryDataService()); - - // Setup a 'filesystem' binary data manager instance - await Container.get(BinaryDataService).init({ - mode: 'filesystem', - availableModes: ['filesystem'], - localStoragePath: temporaryDir, - }); - - // Set our binary data buffer - const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); - const setBinaryDataBufferResponse: IBinaryData = await setBinaryDataBuffer( - { - mimeType: 'txt', - data: 'This should be overwritten with the name of the configured data manager', - }, - inputData, - 'workflowId', - 'executionId', - ); - - // Expect our return object to contain the name of the configured data manager. - expect(setBinaryDataBufferResponse.data).toEqual('filesystem-v2'); - - // Ensure that the input data was successfully persisted to disk. - expect( - readFileSync( - `${temporaryDir}/${setBinaryDataBufferResponse.id?.replace('filesystem-v2:', '')}`, - ), - ).toEqual(inputData); - - // Now, re-fetch our data. - // An ITaskDataConnections object is used to share data between nodes. The top level property, 'main', represents the successful output object from a previous node. - const taskDataConnectionsInput: ITaskDataConnections = { - main: [], - }; - - // We add an input set, with one item at index 0, to this input. It contains an empty json payload and our binary data. - taskDataConnectionsInput.main.push([ - { - json: {}, - binary: { - data: setBinaryDataBufferResponse, - }, - }, - ]); - - // Now, lets fetch our data! The item will be item index 0. - const getBinaryDataBufferResponse: Buffer = await getBinaryDataBuffer( - taskDataConnectionsInput, - 0, - 'data', - 0, - ); - - expect(getBinaryDataBufferResponse).toEqual(inputData); - }); - }); - - describe('parseContentType', () => { - const testCases = [ - { - input: 'text/plain', - expected: { - type: 'text/plain', - parameters: { - charset: 'utf-8', - }, - }, - description: 'should parse basic content type', - }, - { - input: 'TEXT/PLAIN', - expected: { - type: 'text/plain', - parameters: { - charset: 'utf-8', - }, - }, - description: 'should convert type to lowercase', - }, - { - input: 'text/html; charset=iso-8859-1', - expected: { - type: 'text/html', - parameters: { - charset: 'iso-8859-1', - }, - }, - description: 'should parse content type with charset', - }, - { - input: 'application/json; charset=utf-8; boundary=---123', - expected: { - type: 'application/json', - parameters: { - charset: 'utf-8', - boundary: '---123', - }, - }, - description: 'should parse content type with multiple parameters', - }, - { - input: 'text/plain; charset="utf-8"; filename="test.txt"', - expected: { - type: 'text/plain', - parameters: { - charset: 'utf-8', - filename: 'test.txt', - }, - }, - description: 'should handle quoted parameter values', - }, - { - input: 'text/plain; filename=%22test%20file.txt%22', - expected: { - type: 'text/plain', - parameters: { - charset: 'utf-8', - filename: 'test file.txt', - }, - }, - description: 'should handle encoded parameter values', - }, - { - input: undefined, - expected: null, - description: 'should return null for undefined input', - }, - { - input: '', - expected: null, - description: 'should return null for empty string', - }, - ]; - - test.each(testCases)('$description', ({ input, expected }) => { - expect(parseContentType(input)).toEqual(expected); - }); - }); - - describe('parseContentDisposition', () => { - const testCases = [ - { - input: 'attachment; filename="file.txt"', - expected: { type: 'attachment', filename: 'file.txt' }, - description: 'should parse basic content disposition', - }, - { - input: 'attachment; filename=file.txt', - expected: { type: 'attachment', filename: 'file.txt' }, - description: 'should parse filename without quotes', - }, - { - input: 'inline; filename="image.jpg"', - expected: { type: 'inline', filename: 'image.jpg' }, - description: 'should parse inline disposition', - }, - { - input: 'attachment; filename="my file.pdf"', - expected: { type: 'attachment', filename: 'my file.pdf' }, - description: 'should parse filename with spaces', - }, - { - input: "attachment; filename*=UTF-8''my%20file.txt", - expected: { type: 'attachment', filename: 'my file.txt' }, - description: 'should parse filename* parameter (RFC 5987)', - }, - { - input: 'filename="test.txt"', - expected: { type: 'attachment', filename: 'test.txt' }, - description: 'should handle invalid syntax but with filename', - }, - { - input: 'filename=test.txt', - expected: { type: 'attachment', filename: 'test.txt' }, - description: 'should handle invalid syntax with only filename parameter', - }, - { - input: undefined, - expected: null, - description: 'should return null for undefined input', - }, - { - input: '', - expected: null, - description: 'should return null for empty string', - }, - { - input: 'attachment; filename="%F0%9F%98%80.txt"', - expected: { type: 'attachment', filename: '😀.txt' }, - description: 'should handle encoded filenames', - }, - { - input: 'attachment; size=123; filename="test.txt"; creation-date="Thu, 1 Jan 2020"', - expected: { type: 'attachment', filename: 'test.txt' }, - description: 'should handle multiple parameters', - }, - ]; - - test.each(testCases)('$description', ({ input, expected }) => { - expect(parseContentDisposition(input)).toEqual(expected); - }); - }); - - describe('parseIncomingMessage', () => { - it('parses valid content-type header', () => { - const message = mock({ - headers: { 'content-type': 'application/json', 'content-disposition': undefined }, - }); - parseIncomingMessage(message); - - expect(message.contentType).toEqual('application/json'); - }); - - it('parses valid content-type header with parameters', () => { - const message = mock({ - headers: { - 'content-type': 'application/json; charset=utf-8', - 'content-disposition': undefined, - }, - }); - parseIncomingMessage(message); - - expect(message.contentType).toEqual('application/json'); - expect(message.encoding).toEqual('utf-8'); - }); - - it('parses valid content-type header with encoding wrapped in quotes', () => { - const message = mock({ - headers: { - 'content-type': 'application/json; charset="utf-8"', - 'content-disposition': undefined, - }, - }); - parseIncomingMessage(message); - - expect(message.contentType).toEqual('application/json'); - expect(message.encoding).toEqual('utf-8'); - }); - - it('parses valid content-disposition header with filename*', () => { - const message = mock({ - headers: { - 'content-type': undefined, - 'content-disposition': - 'attachment; filename="screenshot%20(1).png"; filename*=UTF-8\'\'screenshot%20(1).png', - }, - }); - parseIncomingMessage(message); - - expect(message.contentDisposition).toEqual({ - filename: 'screenshot (1).png', - type: 'attachment', - }); - }); - - it('parses valid content-disposition header with filename* (quoted)', () => { - const message = mock({ - headers: { - 'content-type': undefined, - 'content-disposition': ' attachment;filename*="utf-8\' \'test-unsplash.jpg"', - }, - }); - parseIncomingMessage(message); - - expect(message.contentDisposition).toEqual({ - filename: 'test-unsplash.jpg', - type: 'attachment', - }); - }); - - it('parses valid content-disposition header with filename and trailing ";"', () => { - const message = mock({ - headers: { - 'content-type': undefined, - 'content-disposition': 'inline; filename="screenshot%20(1).png";', - }, - }); - parseIncomingMessage(message); - - expect(message.contentDisposition).toEqual({ - filename: 'screenshot (1).png', - type: 'inline', - }); - }); - - it('parses non standard content-disposition with missing type', () => { - const message = mock({ - headers: { - 'content-type': undefined, - 'content-disposition': 'filename="screenshot%20(1).png";', - }, - }); - parseIncomingMessage(message); - - expect(message.contentDisposition).toEqual({ - filename: 'screenshot (1).png', - type: 'attachment', - }); - }); - }); - - describe('proxyRequestToAxios', () => { - const baseUrl = 'http://example.de'; - const workflow = mock(); - const hooks = mock(); - const additionalData = mock({ hooks }); - const node = mock(); - - beforeEach(() => { - hooks.executeHookFunctions.mockClear(); - }); - - test('should rethrow an error with `status` property', async () => { - nock(baseUrl).get('/test').reply(400); - - try { - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - } catch (error) { - expect(error.status).toEqual(400); - } - }); - - test('should not throw if the response status is 200', async () => { - nock(baseUrl).get('/test').reply(200); - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - expect(hooks.executeHookFunctions).toHaveBeenCalledWith('nodeFetchedData', [ - workflow.id, - node, - ]); - }); - - test('should throw if the response status is 403', async () => { - const headers = { 'content-type': 'text/plain' }; - nock(baseUrl).get('/test').reply(403, 'Forbidden', headers); - try { - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - } catch (error) { - expect(error.statusCode).toEqual(403); - expect(error.request).toBeUndefined(); - expect(error.response).toMatchObject({ headers, status: 403 }); - expect(error.options).toMatchObject({ - headers: { Accept: '*/*' }, - method: 'get', - url: 'http://example.de/test', - }); - expect(error.config).toBeUndefined(); - expect(error.message).toEqual('403 - "Forbidden"'); - } - expect(hooks.executeHookFunctions).not.toHaveBeenCalled(); - }); - - test('should not throw if the response status is 404, but `simple` option is set to `false`', async () => { - nock(baseUrl).get('/test').reply(404, 'Not Found'); - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/test`, - simple: false, - }); - - expect(response).toEqual('Not Found'); - expect(hooks.executeHookFunctions).toHaveBeenCalledWith('nodeFetchedData', [ - workflow.id, - node, - ]); - }); - - test('should return full response when `resolveWithFullResponse` is set to true', async () => { - nock(baseUrl).get('/test').reply(404, 'Not Found'); - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/test`, - resolveWithFullResponse: true, - simple: false, - }); - - expect(response).toMatchObject({ - body: 'Not Found', - headers: {}, - statusCode: 404, - statusMessage: null, - }); - expect(hooks.executeHookFunctions).toHaveBeenCalledWith('nodeFetchedData', [ - workflow.id, - node, - ]); - }); - - describe('redirects', () => { - test('should forward authorization header', async () => { - nock(baseUrl).get('/redirect').reply(301, '', { Location: 'https://otherdomain.com/test' }); - nock('https://otherdomain.com') - .get('/test') - .reply(200, function () { - return this.req.headers; - }); - - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - auth: { - username: 'testuser', - password: 'testpassword', - }, - headers: { - 'X-Other-Header': 'otherHeaderContent', - }, - resolveWithFullResponse: true, - }); - - expect(response.statusCode).toBe(200); - const forwardedHeaders = JSON.parse(response.body); - expect(forwardedHeaders.authorization).toBe('Basic dGVzdHVzZXI6dGVzdHBhc3N3b3Jk'); - expect(forwardedHeaders['x-other-header']).toBe('otherHeaderContent'); - }); - - test('should follow redirects by default', async () => { - nock(baseUrl) - .get('/redirect') - .reply(301, '', { Location: `${baseUrl}/test` }); - nock(baseUrl).get('/test').reply(200, 'Redirected'); - - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - resolveWithFullResponse: true, - }); - - expect(response).toMatchObject({ - body: 'Redirected', - headers: {}, - statusCode: 200, - }); - }); - - test('should not follow redirects when configured', async () => { - nock(baseUrl) - .get('/redirect') - .reply(301, '', { Location: `${baseUrl}/test` }); - nock(baseUrl).get('/test').reply(200, 'Redirected'); - - await expect( - proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - resolveWithFullResponse: true, - followRedirect: false, - }), - ).rejects.toThrowError(expect.objectContaining({ statusCode: 301 })); - }); - }); - }); - - describe('parseRequestObject', () => { - test('should handle basic request options', async () => { - const axiosOptions = await parseRequestObject({ - url: 'https://example.com', - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: { key: 'value' }, - }); - - expect(axiosOptions).toEqual( - expect.objectContaining({ - url: 'https://example.com', - method: 'POST', - headers: { accept: '*/*', 'content-type': 'application/json' }, - data: { key: 'value' }, - maxRedirects: 0, - }), - ); - }); - - test('should set correct headers for FormData', async () => { - const formData = new FormData(); - formData.append('key', 'value'); - - const axiosOptions = await parseRequestObject({ - url: 'https://example.com', - formData, - headers: { - 'content-type': 'multipart/form-data', - }, - }); - - expect(axiosOptions.headers).toMatchObject({ - accept: '*/*', - 'content-length': 163, - 'content-type': expect.stringMatching(/^multipart\/form-data; boundary=/), - }); - - expect(axiosOptions.data).toBeInstanceOf(FormData); - }); - - test('should not use Host header for SNI', async () => { - const axiosOptions = await parseRequestObject({ - url: 'https://example.de/foo/bar', - headers: { Host: 'other.host.com' }, - }); - expect((axiosOptions.httpsAgent as Agent).options.servername).toEqual('example.de'); - }); - - describe('should set SSL certificates', () => { - const agentOptions: SecureContextOptions = { - ca: '-----BEGIN CERTIFICATE-----\nTEST\n-----END CERTIFICATE-----', - }; - const requestObject: IRequestOptions = { - method: 'GET', - uri: 'https://example.de', - agentOptions, - }; - - test('on regular requests', async () => { - const axiosOptions = await parseRequestObject(requestObject); - expect((axiosOptions.httpsAgent as Agent).options).toEqual({ - servername: 'example.de', - ...agentOptions, - noDelay: true, - path: null, - }); - }); - - test('on redirected requests', async () => { - const axiosOptions = await parseRequestObject(requestObject); - expect(axiosOptions.beforeRedirect).toBeDefined; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const redirectOptions: Record = { agents: {}, hostname: 'example.de' }; - axiosOptions.beforeRedirect!(redirectOptions, mock()); - expect(redirectOptions.agent).toEqual(redirectOptions.agents.https); - expect((redirectOptions.agent as Agent).options).toEqual({ - servername: 'example.de', - ...agentOptions, - noDelay: true, - path: null, - }); - }); - }); - - describe('when followRedirect is true', () => { - test.each(['GET', 'HEAD'] as IHttpRequestMethods[])( - 'should set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followRedirect: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(1234); - }, - ); - - test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'should not set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followRedirect: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(0); - }, - ); - }); - - describe('when followAllRedirects is true', () => { - test.each(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'should set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followAllRedirects: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(1234); - }, - ); - }); - }); - - describe('invokeAxios', () => { - const baseUrl = 'http://example.de'; - - beforeEach(() => { - nock.cleanAll(); - jest.clearAllMocks(); - }); - - it('should throw error for non-401 status codes', async () => { - nock(baseUrl).get('/test').reply(500, {}); - - await expect(invokeAxios({ url: `${baseUrl}/test` })).rejects.toThrow( - 'Request failed with status code 500', - ); - }); - - it('should throw error on 401 without digest auth challenge', async () => { - nock(baseUrl).get('/test').reply(401, {}); - - await expect( - invokeAxios( - { - url: `${baseUrl}/test`, - }, - { sendImmediately: false }, - ), - ).rejects.toThrow('Request failed with status code 401'); - }); - - it('should make successful requests', async () => { - nock(baseUrl).get('/test').reply(200, { success: true }); - - const response = await invokeAxios({ - url: `${baseUrl}/test`, - }); - - expect(response.status).toBe(200); - expect(response.data).toEqual({ success: true }); - }); - - it('should handle digest auth when receiving 401 with nonce', async () => { - nock(baseUrl) - .get('/test') - .matchHeader('authorization', 'Basic dXNlcjpwYXNz') - .once() - .reply(401, {}, { 'www-authenticate': 'Digest realm="test", nonce="abc123", qop="auth"' }); - - nock(baseUrl) - .get('/test') - .matchHeader( - 'authorization', - /^Digest username="user",realm="test",nonce="abc123",uri="\/test",qop="auth",algorithm="MD5",response="[0-9a-f]{32}"/, - ) - .reply(200, { success: true }); - - const response = await invokeAxios( - { - url: `${baseUrl}/test`, - auth: { - username: 'user', - password: 'pass', - }, - }, - { sendImmediately: false }, - ); - - expect(response.status).toBe(200); - expect(response.data).toEqual({ success: true }); - }); - }); - - describe('copyInputItems', () => { - it('should pick only selected properties', () => { - const output = copyInputItems( - [ - { - json: { - a: 1, - b: true, - c: {}, - }, - }, - ], - ['a'], - ); - expect(output).toEqual([{ a: 1 }]); - }); - - it('should convert undefined to null', () => { - const output = copyInputItems( - [ - { - json: { - a: undefined, - }, - }, - ], - ['a'], - ); - expect(output).toEqual([{ a: null }]); - }); - - it('should clone objects', () => { - const input = { - a: { b: 5 }, - }; - const output = copyInputItems( - [ - { - json: input, - }, - ], - ['a'], - ); - expect(output[0].a).toEqual(input.a); - expect(output[0].a === input.a).toEqual(false); - }); - }); - - describe('removeEmptyBody', () => { - test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( - 'Should remove empty body for %s', - async (method) => { - const requestOptions = { - method, - body: {}, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual(undefined); - }, - ); - - test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( - 'Should not remove non-empty body for %s', - async (method) => { - const requestOptions = { - method, - body: { test: true }, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual({ test: true }); - }, - ); - - test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'Should not remove empty body for %s', - async (method) => { - const requestOptions = { - method, - body: {}, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual({}); - }, - ); - }); - - describe('binaryToString', () => { - const ENCODING_SAMPLES = { - utf8: { - text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ', - buffer: Buffer.from([ - 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20, - 0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80, - 0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3, - 0xbc, 0x20, 0xc3, 0xb1, - ]), - }, - - 'iso-8859-15': { - text: 'Café € personnalité', - buffer: Buffer.from([ - 0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61, - 0x6c, 0x69, 0x74, 0xe9, - ]), - }, - - latin1: { - text: 'señor année déjà', - buffer: Buffer.from([ - 0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a, - 0xe0, - ]), - }, - - ascii: { - text: 'Hello, World! 123', - buffer: Buffer.from([ - 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31, - 0x32, 0x33, - ]), - }, - - 'windows-1252': { - text: '€ Smart "quotes" • bullet', - buffer: Buffer.from([ - 0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73, - 0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74, - ]), - }, - - 'shift-jis': { - text: 'こんにちは世界', - buffer: Buffer.from([ - 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45, - ]), - }, - - big5: { - text: '哈囉世界', - buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]), - }, - - 'koi8-r': { - text: 'Привет мир', - buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]), - }, - }; - - describe('should handle Buffer', () => { - for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { - test(`with ${encoding}`, async () => { - const data = await binaryToString(buffer, encoding); - expect(data).toBe(text); - }); - } - }); - - describe('should handle streams', () => { - for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { - test(`with ${encoding}`, async () => { - const stream = Readable.from(buffer); - const data = await binaryToString(stream, encoding); - expect(data).toBe(text); - }); - } - }); - - describe('should handle IncomingMessage', () => { - for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { - test(`with ${encoding}`, async () => { - const response = Readable.from(buffer) as IncomingMessage; - response.headers = { 'content-type': `application/json;charset=${encoding}` }; - // @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks - response.__proto__ = IncomingMessage.prototype; - const data = await binaryToString(response); - expect(data).toBe(text); - }); - } - }); - }); -}); - -describe('isFilePathBlocked', () => { - test('should return true for static cache dir', () => { - const filePath = Container.get(InstanceSettings).staticCacheDir; - - expect(isFilePathBlocked(filePath)).toBe(true); - }); -}); diff --git a/packages/core/test/helpers/index.ts b/packages/core/test/helpers/index.ts index 14f19789b0..27008b177a 100644 --- a/packages/core/test/helpers/index.ts +++ b/packages/core/test/helpers/index.ts @@ -6,7 +6,6 @@ import type { INodeType, INodeTypes, IRun, - ITaskData, IVersionedNodeType, IWorkflowBase, IWorkflowExecuteAdditionalData, @@ -14,10 +13,11 @@ import type { WorkflowTestData, INodeTypeData, } from 'n8n-workflow'; -import { ApplicationError, NodeHelpers, WorkflowHooks } from 'n8n-workflow'; +import { ApplicationError, NodeHelpers } from 'n8n-workflow'; import path from 'path'; import { UnrecognizedNodeTypeError } from '@/errors'; +import { ExecutionLifecycleHooks } from '@/execution-engine/execution-lifecycle-hooks'; import { predefinedNodesTypes } from './constants'; @@ -53,22 +53,12 @@ export function WorkflowExecuteAdditionalData( waitPromise: IDeferredPromise, nodeExecutionOrder: string[], ): IWorkflowExecuteAdditionalData { - const hookFunctions = { - nodeExecuteAfter: [ - async (nodeName: string, _data: ITaskData): Promise => { - nodeExecutionOrder.push(nodeName); - }, - ], - workflowExecuteAfter: [ - async (fullRunData: IRun): Promise => { - waitPromise.resolve(fullRunData); - }, - ], - }; - - return mock({ - hooks: new WorkflowHooks(hookFunctions, 'trigger', '1', mock()), + const hooks = new ExecutionLifecycleHooks('trigger', '1', mock()); + hooks.addHandler('nodeExecuteAfter', (nodeName) => { + nodeExecutionOrder.push(nodeName); }); + hooks.addHandler('workflowExecuteAfter', (fullRunData) => waitPromise.resolve(fullRunData)); + return mock({ hooks }); } const preparePinData = (pinData: IDataObject) => { @@ -85,8 +75,7 @@ const preparePinData = (pinData: IDataObject) => { return returnData; }; -const readJsonFileSync = (filePath: string) => - JSON.parse(readFileSync(path.join(BASE_DIR, filePath), 'utf-8')) as T; +const readJsonFileSync = (filePath: string) => JSON.parse(readFileSync(filePath, 'utf-8')) as T; export function getNodeTypes(testData: WorkflowTestData[] | WorkflowTestData) { if (!Array.isArray(testData)) { @@ -100,7 +89,7 @@ export function getNodeTypes(testData: WorkflowTestData[] | WorkflowTestData) { const nodeNames = nodes.map((n) => n.type); const knownNodes = readJsonFileSync>( - 'nodes-base/dist/known/nodes.json', + path.join(BASE_DIR, 'nodes-base/dist/known/nodes.json'), ); for (const nodeName of nodeNames) { @@ -120,14 +109,14 @@ export function getNodeTypes(testData: WorkflowTestData[] | WorkflowTestData) { return nodeTypes; } -const getWorkflowFilenames = (dirname: string, testFolder = 'workflows') => { +const getWorkflowFilepaths = (dirname: string, testFolder = 'workflows') => { const workflows: string[] = []; const filenames: string[] = readdirSync(`${dirname}${path.sep}${testFolder}`); filenames.forEach((file) => { if (file.endsWith('.json')) { - workflows.push(path.join('core', 'test', testFolder, file)); + workflows.push(path.join(dirname, testFolder, file)); } }); @@ -135,11 +124,11 @@ const getWorkflowFilenames = (dirname: string, testFolder = 'workflows') => { }; export const workflowToTests = (dirname: string, testFolder = 'workflows') => { - const workflowFiles: string[] = getWorkflowFilenames(dirname, testFolder); + const workflowFilepaths: string[] = getWorkflowFilepaths(dirname, testFolder); const testCases: WorkflowTestData[] = []; - for (const filePath of workflowFiles) { + for (const filePath of workflowFilepaths) { const description = filePath.replace('.json', ''); const workflowData = readJsonFileSync(filePath); if (workflowData.pinData === undefined) { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 111dd828dd..0fbcc8b81a 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -4,7 +4,8 @@ "rootDir": ".", "baseUrl": "src", "paths": { - "@/*": ["./*"] + "@/*": ["./*"], + "@test/*": ["../test/*"] }, "tsBuildInfoFile": "dist/typecheck.tsbuildinfo", "emitDecoratorMetadata": true, diff --git a/packages/design-system/.eslintrc.js b/packages/design-system/.eslintrc.js index deece11c63..99d5ff4c6e 100644 --- a/packages/design-system/.eslintrc.js +++ b/packages/design-system/.eslintrc.js @@ -1,13 +1,9 @@ -const sharedOptions = require('@n8n_io/eslint-config/shared'); +const { createFrontendEslintConfig } = require('@n8n/frontend-eslint-config'); /** * @type {import('@types/eslint').ESLint.ConfigData} */ -module.exports = { - extends: ['@n8n_io/eslint-config/frontend'], - - ...sharedOptions(__dirname, 'frontend'), - +module.exports = createFrontendEslintConfig(__dirname, { rules: { // TODO: Remove these 'import/no-default-export': 'warn', @@ -44,4 +40,4 @@ module.exports = { }, }, ], -}; +}); diff --git a/packages/design-system/package.json b/packages/design-system/package.json index d35d701825..3ddc30ebbe 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -1,8 +1,8 @@ { "name": "n8n-design-system", - "version": "1.64.0", - "main": "src/main.ts", - "import": "src/main.ts", + "version": "1.68.0", + "main": "src/index.ts", + "import": "src/index.ts", "scripts": { "dev": "pnpm run storybook", "clean": "rimraf dist .turbo", @@ -19,9 +19,12 @@ "lintfix": "eslint src --ext .js,.ts,.vue --fix" }, "devDependencies": { + "@n8n/frontend-eslint-config": "workspace:*", + "@n8n/frontend-typescript-config": "workspace:*", + "@n8n/frontend-vitest-config": "workspace:*", "@n8n/storybook": "workspace:*", - "@testing-library/jest-dom": "^6.5.0", - "@testing-library/user-event": "^14.5.2", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/user-event": "^14.6.0", "@testing-library/vue": "^8.1.0", "@types/markdown-it": "^13.0.9", "@types/markdown-it-emoji": "^2.0.2", @@ -41,6 +44,7 @@ "vue-tsc": "catalog:frontend" }, "dependencies": { + "@n8n/composables": "workspace:*", "@fortawesome/fontawesome-svg-core": "^1.2.36", "@fortawesome/free-solid-svg-icons": "^5.15.4", "@fortawesome/vue-fontawesome": "^3.0.3", diff --git a/packages/design-system/src/components/N8nActionBox/ActionBox.vue b/packages/design-system/src/components/N8nActionBox/ActionBox.vue index 02de38143c..b04ccb1049 100644 --- a/packages/design-system/src/components/N8nActionBox/ActionBox.vue +++ b/packages/design-system/src/components/N8nActionBox/ActionBox.vue @@ -10,8 +10,8 @@ import N8nText from '../N8nText'; interface ActionBoxProps { emoji: string; heading: string; - buttonText: string; - buttonType: ButtonType; + buttonText?: string; + buttonType?: ButtonType; buttonDisabled?: boolean; buttonIcon?: string; description: string; diff --git a/packages/design-system/src/components/N8nCheckbox/__snapshots__/Checkbox.test.ts.snap b/packages/design-system/src/components/N8nCheckbox/__snapshots__/Checkbox.test.ts.snap index c6dd12ec37..323ba7ae4f 100644 --- a/packages/design-system/src/components/N8nCheckbox/__snapshots__/Checkbox.test.ts.snap +++ b/packages/design-system/src/components/N8nCheckbox/__snapshots__/Checkbox.test.ts.snap @@ -31,25 +31,39 @@ exports[`components > N8nCheckbox > should render with both child and label 1`] class="container" data-test-id="input-label" > - - - - + @@ -122,25 +136,39 @@ exports[`components > N8nCheckbox > should render with label 1`] = ` class="container" data-test-id="input-label" > - - - - + diff --git a/packages/design-system/src/components/N8nDatatable/Datatable.vue b/packages/design-system/src/components/N8nDatatable/Datatable.vue index 905aef4671..d3447229d1 100644 --- a/packages/design-system/src/components/N8nDatatable/Datatable.vue +++ b/packages/design-system/src/components/N8nDatatable/Datatable.vue @@ -1,5 +1,5 @@ - diff --git a/packages/design-system/src/components/N8nSelect/Select.vue b/packages/design-system/src/components/N8nSelect/Select.vue index 80a065d429..28c1fc049a 100644 --- a/packages/design-system/src/components/N8nSelect/Select.vue +++ b/packages/design-system/src/components/N8nSelect/Select.vue @@ -136,6 +136,12 @@ defineExpose({ + + diff --git a/packages/design-system/src/components/N8nSelectableList/SelectableList.stories.ts b/packages/design-system/src/components/N8nSelectableList/SelectableList.stories.ts new file mode 100644 index 0000000000..a4f5244143 --- /dev/null +++ b/packages/design-system/src/components/N8nSelectableList/SelectableList.stories.ts @@ -0,0 +1,45 @@ +import type { StoryFn } from '@storybook/vue3'; + +import N8nSelectableList from './SelectableList.vue'; + +export default { + title: 'Modules/SelectableList', + component: N8nSelectableList, + argTypes: {}, + parameters: { + backgrounds: { default: '--color-background-light' }, + }, +}; + +const Template: StoryFn = (args, { argTypes }) => ({ + setup: () => ({ + args: { ...args, modelValue: undefined }, + model: args.modelValue, + }), + props: Object.keys(argTypes), + // Generics make this difficult to type + components: N8nSelectableList as never, + template: + '', +}); + +export const SelectableList = Template.bind({}); +SelectableList.args = { + modelValue: { + propC: 'propC pre-existing initial value', + }, + inputs: [ + { + name: 'propC', + initialValue: 'propC default', + }, + { + name: 'propB', + initialValue: 0, + }, + { + name: 'propA', + initialValue: false, + }, + ], +}; diff --git a/packages/design-system/src/components/N8nSelectableList/SelectableList.test.ts b/packages/design-system/src/components/N8nSelectableList/SelectableList.test.ts new file mode 100644 index 0000000000..21f8040699 --- /dev/null +++ b/packages/design-system/src/components/N8nSelectableList/SelectableList.test.ts @@ -0,0 +1,97 @@ +import { fireEvent, render } from '@testing-library/vue'; + +import N8nSelectableList from './SelectableList.vue'; + +describe('N8nSelectableList', () => { + it('renders when empty', () => { + const wrapper = render(N8nSelectableList, { + props: { + modelValue: {}, + inputs: [], + }, + }); + expect(wrapper.html()).toMatchSnapshot(); + }); + + it('renders one clickable element that can be added and removed', async () => { + const wrapper = render(N8nSelectableList, { + props: { + modelValue: {}, + inputs: [{ name: 'propA', initialValue: '' }], + }, + }); + + expect(wrapper.getByTestId('selectable-list-selectable-propA')).toBeInTheDocument(); + + await fireEvent.click(wrapper.getByTestId('selectable-list-selectable-propA')); + + expect(wrapper.queryByTestId('selectable-list-selectable-propA')).not.toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-slot-propA')).toBeInTheDocument(); + + await fireEvent.click(wrapper.getByTestId('selectable-list-remove-slot-propA')); + + expect(wrapper.queryByTestId('selectable-list-slot-propA')).not.toBeInTheDocument(); + }); + + it('renders multiple elements with some pre-selected', () => { + const wrapper = render(N8nSelectableList, { + props: { + modelValue: { + propC: false, + propA: 'propA value', + }, + inputs: [ + { name: 'propD', initialValue: true }, + { name: 'propC', initialValue: true }, + { name: 'propB', initialValue: 3 }, + { name: 'propA', initialValue: '' }, + ], + }, + }); + + expect(wrapper.queryByTestId('selectable-list-selectable-propA')).not.toBeInTheDocument(); + expect(wrapper.queryByTestId('selectable-list-selectable-propC')).not.toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-slot-propA')).toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-selectable-propB')).toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-slot-propC')).toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-selectable-propD')).toBeInTheDocument(); + + // This asserts order - specifically that propA appears before propC + expect( + wrapper + .getByTestId('selectable-list-slot-propA') + .compareDocumentPosition(wrapper.getByTestId('selectable-list-slot-propC')), + ).toEqual(4); + + expect( + wrapper + .getByTestId('selectable-list-selectable-propB') + .compareDocumentPosition(wrapper.getByTestId('selectable-list-selectable-propD')), + ).toEqual(4); + + expect(wrapper.html()).toMatchSnapshot(); + }); + + it('renders disabled collection without selectables', async () => { + const wrapper = render(N8nSelectableList, { + props: { + modelValue: { + propB: 'propB value', + }, + disabled: true, + inputs: [ + { name: 'propA', initialValue: '' }, + { name: 'propB', initialValue: '' }, + { name: 'propC', initialValue: '' }, + ], + }, + }); + + expect(wrapper.queryByTestId('selectable-list-selectable-propA')).not.toBeInTheDocument(); + expect(wrapper.getByTestId('selectable-list-slot-propB')).toBeInTheDocument(); + expect(wrapper.queryByTestId('selectable-list-selectable-propB')).not.toBeInTheDocument(); + expect(wrapper.queryByTestId('selectable-list-selectable-propC')).not.toBeInTheDocument(); + + expect(wrapper.html()).toMatchSnapshot(); + }); +}); diff --git a/packages/design-system/src/components/N8nSelectableList/SelectableList.vue b/packages/design-system/src/components/N8nSelectableList/SelectableList.vue new file mode 100644 index 0000000000..de0133d2d0 --- /dev/null +++ b/packages/design-system/src/components/N8nSelectableList/SelectableList.vue @@ -0,0 +1,146 @@ + + + + + diff --git a/packages/design-system/src/components/N8nSelectableList/__snapshots__/SelectableList.test.ts.snap b/packages/design-system/src/components/N8nSelectableList/__snapshots__/SelectableList.test.ts.snap new file mode 100644 index 0000000000..63397a468b --- /dev/null +++ b/packages/design-system/src/components/N8nSelectableList/__snapshots__/SelectableList.test.ts.snap @@ -0,0 +1,28 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`N8nSelectableList > renders disabled collection without selectables 1`] = ` +"
+ +
+
+
+
" +`; + +exports[`N8nSelectableList > renders multiple elements with some pre-selected 1`] = ` +"
+
+ Add a propB
+ Add a propD
+
+
+
+
+
+
+
" +`; + +exports[`N8nSelectableList > renders when empty 1`] = ` +"
+
+
" +`; diff --git a/packages/design-system/src/components/N8nSelectableList/index.ts b/packages/design-system/src/components/N8nSelectableList/index.ts new file mode 100644 index 0000000000..6c2d0e1127 --- /dev/null +++ b/packages/design-system/src/components/N8nSelectableList/index.ts @@ -0,0 +1,3 @@ +import N8nSelectableList from './SelectableList.vue'; + +export default N8nSelectableList; diff --git a/packages/design-system/src/components/N8nUserStack/UserStack.test.ts b/packages/design-system/src/components/N8nUserStack/UserStack.test.ts index 1b5ae600ae..5e2d1e10fa 100644 --- a/packages/design-system/src/components/N8nUserStack/UserStack.test.ts +++ b/packages/design-system/src/components/N8nUserStack/UserStack.test.ts @@ -1,8 +1,7 @@ import { render } from '@testing-library/vue'; -import { N8nAvatar, N8nUserInfo } from 'n8n-design-system/main'; - import UserStack from './UserStack.vue'; +import { N8nAvatar, N8nUserInfo } from '../index'; describe('UserStack', () => { it('should render flat user list', () => { diff --git a/packages/design-system/src/components/index.ts b/packages/design-system/src/components/index.ts index ce4360fe46..6b584b8e8e 100644 --- a/packages/design-system/src/components/index.ts +++ b/packages/design-system/src/components/index.ts @@ -33,6 +33,7 @@ export { default as N8nNodeCreatorNode } from './N8nNodeCreatorNode'; export { default as N8nNodeIcon } from './N8nNodeIcon'; export { default as N8nNotice } from './N8nNotice'; export { default as N8nOption } from './N8nOption'; +export { default as N8nSelectableList } from './N8nSelectableList'; export { default as N8nPopover } from './N8nPopover'; export { default as N8nPulse } from './N8nPulse'; export { default as N8nRadioButtons } from './N8nRadioButtons'; diff --git a/packages/design-system/src/css/_tokens.dark.scss b/packages/design-system/src/css/_tokens.dark.scss index 00eeafb981..051de41601 100644 --- a/packages/design-system/src/css/_tokens.dark.scss +++ b/packages/design-system/src/css/_tokens.dark.scss @@ -227,6 +227,7 @@ --color-notification-background: var(--prim-gray-740); // Execution + --execution-card-background: var(--color-foreground-light); --execution-card-background-hover: var(--color-foreground-base); --execution-selector-background: var(--prim-gray-740); --execution-selector-text: var(--color-text-base); diff --git a/packages/design-system/src/css/_tokens.scss b/packages/design-system/src/css/_tokens.scss index 3b6b7451be..50c98ac2ee 100644 --- a/packages/design-system/src/css/_tokens.scss +++ b/packages/design-system/src/css/_tokens.scss @@ -37,6 +37,8 @@ // Danger --color-danger-shade-1: var(--prim-color-alt-c-shade-100); --color-danger: var(--prim-color-alt-c); + --color-danger-light: var(--prim-color-alt-c-tint-150); + --color-danger-light-2: var(--prim-color-alt-c-tint-250); --color-danger-tint-1: var(--prim-color-alt-c-tint-400); --color-danger-tint-2: var(--prim-color-alt-c-tint-450); @@ -287,6 +289,7 @@ --execution-card-border-waiting: var(--prim-color-secondary-tint-300); --execution-card-border-running: var(--prim-color-alt-b-tint-250); --execution-card-border-unknown: var(--prim-gray-120); + --execution-card-background: var(--color-foreground-xlight); --execution-card-background-hover: var(--color-foreground-light); --execution-card-text-waiting: var(--color-secondary); --execution-selector-background: var(--color-background-dark); diff --git a/packages/design-system/src/main.ts b/packages/design-system/src/index.ts similarity index 74% rename from packages/design-system/src/main.ts rename to packages/design-system/src/index.ts index dddb89f888..d96d74532f 100644 --- a/packages/design-system/src/main.ts +++ b/packages/design-system/src/index.ts @@ -1,6 +1,5 @@ import * as locale from './locale'; -export { useDeviceSupport } from './composables/useDeviceSupport'; export * from './components'; export * from './plugin'; export * from './types'; diff --git a/packages/design-system/src/locale/lang/en.ts b/packages/design-system/src/locale/lang/en.ts index 35f2fd9f0e..3743bf92c6 100644 --- a/packages/design-system/src/locale/lang/en.ts +++ b/packages/design-system/src/locale/lang/en.ts @@ -51,4 +51,5 @@ export default { 'iconPicker.button.defaultToolTip': 'Choose icon', 'iconPicker.tabs.icons': 'Icons', 'iconPicker.tabs.emojis': 'Emojis', + 'selectableList.addDefault': '+ Add a', } as N8nLocale; diff --git a/packages/design-system/tsconfig.json b/packages/design-system/tsconfig.json index 1b96b238c1..4a3a72f3fc 100644 --- a/packages/design-system/tsconfig.json +++ b/packages/design-system/tsconfig.json @@ -1,16 +1,10 @@ { - "extends": "../../tsconfig.json", + "extends": "@n8n/frontend-typescript-config", "compilerOptions": { - "rootDir": ".", - "outDir": "dist", - "target": "esnext", - "module": "esnext", - "importHelpers": true, - "allowJs": true, - "incremental": false, - "allowSyntheticDefaultImports": true, "baseUrl": ".", - "types": ["vitest/globals"], + "rootDirs": [".", "../../frontend/@n8n/composables/src"], + "outDir": "dist", + "types": ["vite/client", "vitest/globals"], "typeRoots": [ "./node_modules/@testing-library", "./node_modules/@types", @@ -18,12 +12,9 @@ "../../node_modules/@types" ], "paths": { - "n8n-design-system/*": ["./src/*"] - }, - "lib": ["esnext", "dom", "dom.iterable", "scripthost"], - // TODO: remove all options below this line - "noImplicitAny": false, - "noImplicitReturns": false + "n8n-design-system*": ["./src*"], + "@n8n/composables*": ["../frontend/@n8n/composables/src*"] + } }, "include": ["src/**/*.ts", "src/**/*.vue"] } diff --git a/packages/design-system/vite.config.mts b/packages/design-system/vite.config.mts index 783218b793..1a6c504c02 100644 --- a/packages/design-system/vite.config.mts +++ b/packages/design-system/vite.config.mts @@ -1,35 +1,12 @@ import vue from '@vitejs/plugin-vue'; import { resolve } from 'path'; import { defineConfig, mergeConfig } from 'vite'; -import { type UserConfig } from 'vitest'; -import { defineConfig as defineVitestConfig } from 'vitest/config'; import components from 'unplugin-vue-components/vite'; import icons from 'unplugin-icons/vite'; import iconsResolver from 'unplugin-icons/resolver'; +import { vitestConfig } from '@n8n/frontend-vitest-config'; -export const vitestConfig = defineVitestConfig({ - test: { - silent: true, - globals: true, - environment: 'jsdom', - setupFiles: ['./src/__tests__/setup.ts'], - ...(process.env.COVERAGE_ENABLED === 'true' - ? { - coverage: { - enabled: true, - provider: 'v8', - reporter: process.env.CI === 'true' ? 'cobertura' : 'text-summary', - all: true, - }, - } - : {}), - css: { - modules: { - classNameStrategy: 'non-scoped', - }, - }, - }, -}) as UserConfig; +const frontendDir = resolve(__dirname, '..', 'frontend'); export default mergeConfig( defineConfig({ @@ -53,12 +30,13 @@ export default mergeConfig( alias: { '@': resolve(__dirname, 'src'), 'n8n-design-system': resolve(__dirname, 'src'), + '@n8n/composables(.*)': resolve(frontendDir, '@n8n', 'composables', 'src$1'), lodash: 'lodash-es', }, }, build: { lib: { - entry: resolve(__dirname, 'src', 'main.ts'), + entry: resolve(__dirname, 'src', 'index.ts'), name: 'N8nDesignSystem', fileName: (format) => `n8n-design-system.${format}.js`, }, diff --git a/packages/editor-ui/.eslintrc.js b/packages/editor-ui/.eslintrc.js index 96dcf7d915..a32d3a3d62 100644 --- a/packages/editor-ui/.eslintrc.js +++ b/packages/editor-ui/.eslintrc.js @@ -1,13 +1,9 @@ -const sharedOptions = require('@n8n_io/eslint-config/shared'); +const { createFrontendEslintConfig } = require('@n8n/frontend-eslint-config'); /** * @type {import('@types/eslint').ESLint.ConfigData} */ -module.exports = { - extends: ['@n8n_io/eslint-config/frontend'], - - ...sharedOptions(__dirname, 'frontend'), - +module.exports = createFrontendEslintConfig(__dirname, { rules: { 'n8n-local-rules/dangerously-use-html-string-missing': 'error', @@ -41,4 +37,4 @@ module.exports = { '@typescript-eslint/no-redundant-type-constituents': 'warn', '@typescript-eslint/no-unsafe-enum-comparison': 'warn', }, -}; +}); diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index ed37d52ca5..5978a2e453 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -1,6 +1,6 @@ { "name": "n8n-editor-ui", - "version": "1.74.1", + "version": "1.78.0", "description": "Workflow Editor UI for n8n", "main": "index.js", "scripts": { @@ -39,14 +39,15 @@ "@n8n/chat": "workspace:*", "@n8n/codemirror-lang": "workspace:*", "@n8n/codemirror-lang-sql": "^1.0.2", + "@n8n/composables": "workspace:*", "@n8n/permissions": "workspace:*", "@replit/codemirror-indentation-markers": "^6.5.3", - "@typescript/vfs": "^1.6.0", "@sentry/vue": "catalog:frontend", + "@typescript/vfs": "^1.6.0", "@vue-flow/background": "^1.3.2", "@vue-flow/controls": "^1.1.2", - "@vue-flow/core": "^1.41.6", - "@vue-flow/minimap": "^1.5.0", + "@vue-flow/core": "^1.42.1", + "@vue-flow/minimap": "^1.5.2", "@vue-flow/node-resizer": "^1.4.0", "@vueuse/components": "^10.11.0", "@vueuse/core": "^10.11.0", @@ -55,7 +56,9 @@ "change-case": "^5.4.4", "chart.js": "^4.4.0", "codemirror-lang-html-n8n": "^1.0.0", + "curlconverter": "^4.12.0", "comlink": "^4.4.1", + "core-js": "^3.40.0", "dateformat": "^3.0.3", "email-providers": "^2.0.1", "esprima-next": "5.8.4", @@ -87,16 +90,21 @@ "vue-router": "catalog:frontend", "vue-virtual-scroller": "2.0.0-beta.8", "vue3-touch-events": "^4.1.3", + "web-tree-sitter": "0.24.3", "vuedraggable": "4.1.0", "xss": "catalog:" }, "devDependencies": { + "@n8n/frontend-eslint-config": "workspace:*", + "@n8n/frontend-vitest-config": "workspace:*", + "@n8n/frontend-typescript-config": "workspace:*", "@faker-js/faker": "^8.0.2", "@iconify/json": "^2.2.228", "@pinia/testing": "^0.1.6", "@types/dateformat": "^3.0.0", "@types/file-saver": "^2.0.1", "@types/humanize-duration": "^3.27.1", + "@types/json-schema": "^7.0.15", "@types/jsonpath": "^0.2.0", "@types/lodash-es": "^4.17.6", "@types/luxon": "^3.2.0", @@ -109,6 +117,7 @@ "unplugin-icons": "^0.19.0", "unplugin-vue-components": "^0.27.2", "vite": "catalog:frontend", + "vite-plugin-static-copy": "2.2.0", "vite-svg-loader": "5.1.0", "vitest": "catalog:frontend", "vitest-mock-extended": "catalog:frontend", diff --git a/packages/editor-ui/public/static/n8n-logo.png b/packages/editor-ui/public/static/n8n-logo.png new file mode 100644 index 0000000000..2bb6b2d750 Binary files /dev/null and b/packages/editor-ui/public/static/n8n-logo.png differ diff --git a/packages/editor-ui/public/static/og_image.png b/packages/editor-ui/public/static/og_image.png new file mode 100644 index 0000000000..b5e13a5424 Binary files /dev/null and b/packages/editor-ui/public/static/og_image.png differ diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 6cdd045703..7f7e445fde 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -54,6 +54,7 @@ import type { REGULAR_NODE_CREATOR_VIEW, AI_OTHERS_NODE_CREATOR_VIEW, ROLE, + AI_UNCATEGORIZED_CATEGORY, } from '@/constants'; import type { BulkCommand, Undoable } from '@/models/history'; @@ -404,6 +405,7 @@ export interface IExecutionResponse extends IExecutionBase { data?: IRunExecutionData; workflowData: IWorkflowDb; executedNode?: string; + triggerNode?: string; } export type ExecutionSummaryWithScopes = ExecutionSummary & { scopes: Scope[] }; @@ -740,6 +742,8 @@ export interface CreateElementBase { export interface NodeCreateElement extends CreateElementBase { type: 'node'; subcategory: string; + resource?: string; + operation?: string; properties: SimplifiedNodeType; } @@ -886,7 +890,6 @@ export interface RootState { endpointWebhook: string; endpointWebhookTest: string; endpointWebhookWaiting: string; - pushConnectionActive: boolean; timezone: string; executionTimeout: number; maxExecutionTimeout: number; @@ -1010,7 +1013,8 @@ export type NodeFilterType = | typeof REGULAR_NODE_CREATOR_VIEW | typeof TRIGGER_NODE_CREATOR_VIEW | typeof AI_NODE_CREATOR_VIEW - | typeof AI_OTHERS_NODE_CREATOR_VIEW; + | typeof AI_OTHERS_NODE_CREATOR_VIEW + | typeof AI_UNCATEGORIZED_CATEGORY; export type NodeCreatorOpenSource = | '' @@ -1484,14 +1488,6 @@ export interface IN8nPromptResponse { updated: boolean; } -export type ApiKey = { - id: string; - label: string; - apiKey: string; - createdAt: string; - updatedAt: string; -}; - export type InputPanel = { nodeName?: string; run?: number; diff --git a/packages/editor-ui/src/__tests__/defaults.ts b/packages/editor-ui/src/__tests__/defaults.ts index 46e35a7d17..f1d16ed7e3 100644 --- a/packages/editor-ui/src/__tests__/defaults.ts +++ b/packages/editor-ui/src/__tests__/defaults.ts @@ -62,7 +62,13 @@ export const defaultSettings: FrontendSettings = { disableSessionRecording: false, enabled: false, }, - publicApi: { enabled: false, latestVersion: 0, path: '', swaggerUi: { enabled: false } }, + publicApi: { + apiKeysPerUserLimit: 0, + enabled: false, + latestVersion: 0, + path: '', + swaggerUi: { enabled: false }, + }, pushBackend: 'websocket', saveDataErrorExecution: 'all', saveDataSuccessExecution: 'all', @@ -131,4 +137,8 @@ export const defaultSettings: FrontendSettings = { }, betaFeatures: [], easyAIWorkflowOnboarded: false, + partialExecution: { + version: 1, + enforce: false, + }, }; diff --git a/packages/editor-ui/src/__tests__/mocks.ts b/packages/editor-ui/src/__tests__/mocks.ts index 15688de9ff..f514d809d9 100644 --- a/packages/editor-ui/src/__tests__/mocks.ts +++ b/packages/editor-ui/src/__tests__/mocks.ts @@ -54,7 +54,7 @@ export const mockNodeTypeDescription = ({ credentials = [], inputs = [NodeConnectionType.Main], outputs = [NodeConnectionType.Main], - codex = {}, + codex = undefined, properties = [], }: { name?: INodeTypeDescription['name']; diff --git a/packages/editor-ui/src/__tests__/setup.ts b/packages/editor-ui/src/__tests__/setup.ts index 35bfe7aafb..934017e2ba 100644 --- a/packages/editor-ui/src/__tests__/setup.ts +++ b/packages/editor-ui/src/__tests__/setup.ts @@ -1,5 +1,6 @@ import '@testing-library/jest-dom'; import { configure } from '@testing-library/vue'; +import 'core-js/proposals/set-methods-v2'; configure({ testIdAttribute: 'data-test-id' }); @@ -61,3 +62,25 @@ Object.defineProperty(window, 'matchMedia', { dispatchEvent: vi.fn(), })), }); + +class Worker { + onmessage: (message: string) => void; + + url: string; + + constructor(url: string) { + this.url = url; + this.onmessage = () => {}; + } + + postMessage(message: string) { + this.onmessage(message); + } + + addEventListener() {} +} + +Object.defineProperty(window, 'Worker', { + writable: true, + value: Worker, +}); diff --git a/packages/editor-ui/src/api/ai.ts b/packages/editor-ui/src/api/ai.ts index 04c08c78f2..a6721d2652 100644 --- a/packages/editor-ui/src/api/ai.ts +++ b/packages/editor-ui/src/api/ai.ts @@ -1,6 +1,9 @@ +import { useAIAssistantHelpers } from '@/composables/useAIAssistantHelpers'; +import { AI_ASSISTANT_MAX_CONTENT_LENGTH } from '@/constants'; import type { ICredentialsResponse, IRestApiContext } from '@/Interface'; import type { AskAiRequest, ChatRequest, ReplaceCodeRequest } from '@/types/assistant.types'; import { makeRestApiRequest, streamRequest } from '@/utils/apiUtils'; +import { getObjectSizeInKB } from '@/utils/objectUtils'; import type { IDataObject } from 'n8n-workflow'; export function chatWithAssistant( @@ -10,6 +13,15 @@ export function chatWithAssistant( onDone: () => void, onError: (e: Error) => void, ): void { + try { + const payloadSize = getObjectSizeInKB(payload.payload); + if (payloadSize > AI_ASSISTANT_MAX_CONTENT_LENGTH) { + useAIAssistantHelpers().trimPayloadSize(payload); + } + } catch (e) { + onError(e); + return; + } void streamRequest( ctx, '/ai/chat', diff --git a/packages/editor-ui/src/api/api-keys.ts b/packages/editor-ui/src/api/api-keys.ts index b4b44c8e13..c9af96f136 100644 --- a/packages/editor-ui/src/api/api-keys.ts +++ b/packages/editor-ui/src/api/api-keys.ts @@ -1,12 +1,21 @@ -import type { ApiKey, IRestApiContext } from '@/Interface'; +import type { IRestApiContext } from '@/Interface'; import { makeRestApiRequest } from '@/utils/apiUtils'; +import type { + CreateApiKeyRequestDto, + UpdateApiKeyRequestDto, + ApiKey, + ApiKeyWithRawValue, +} from '@n8n/api-types'; export async function getApiKeys(context: IRestApiContext): Promise { return await makeRestApiRequest(context, 'GET', '/api-keys'); } -export async function createApiKey(context: IRestApiContext): Promise { - return await makeRestApiRequest(context, 'POST', '/api-keys'); +export async function createApiKey( + context: IRestApiContext, + payload: CreateApiKeyRequestDto, +): Promise { + return await makeRestApiRequest(context, 'POST', '/api-keys', payload); } export async function deleteApiKey( @@ -15,3 +24,11 @@ export async function deleteApiKey( ): Promise<{ success: boolean }> { return await makeRestApiRequest(context, 'DELETE', `/api-keys/${id}`); } + +export async function updateApiKey( + context: IRestApiContext, + id: string, + payload: UpdateApiKeyRequestDto, +): Promise<{ success: boolean }> { + return await makeRestApiRequest(context, 'PATCH', `/api-keys/${id}`, payload); +} diff --git a/packages/editor-ui/src/api/credentials.ts b/packages/editor-ui/src/api/credentials.ts index 6ddb37c16a..18bb7b3f3d 100644 --- a/packages/editor-ui/src/api/credentials.ts +++ b/packages/editor-ui/src/api/credentials.ts @@ -32,6 +32,7 @@ export async function getAllCredentials( ): Promise { return await makeRestApiRequest(context, 'GET', '/credentials', { ...(includeScopes ? { includeScopes } : {}), + includeData: true, ...(filter ? { filter } : {}), }); } diff --git a/packages/editor-ui/src/api/curlHelper.ts b/packages/editor-ui/src/api/curlHelper.ts deleted file mode 100644 index 5c09ab0b9a..0000000000 --- a/packages/editor-ui/src/api/curlHelper.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { CurlToJSONResponse, IRestApiContext } from '@/Interface'; -import { makeRestApiRequest } from '@/utils/apiUtils'; - -export async function getCurlToJson( - context: IRestApiContext, - curlCommand: string, -): Promise { - return await makeRestApiRequest(context, 'POST', '/curl/to-json', { curlCommand }); -} diff --git a/packages/editor-ui/src/api/projects.api.ts b/packages/editor-ui/src/api/projects.api.ts index 325feefc63..28e4f8abf7 100644 --- a/packages/editor-ui/src/api/projects.api.ts +++ b/packages/editor-ui/src/api/projects.api.ts @@ -1,21 +1,14 @@ import type { IRestApiContext } from '@/Interface'; import { makeRestApiRequest } from '@/utils/apiUtils'; -import type { - Project, - ProjectCreateRequest, - ProjectListItem, - ProjectUpdateRequest, - ProjectsCount, -} from '@/types/projects.types'; +import type { Project, ProjectListItem, ProjectsCount } from '@/types/projects.types'; +import type { CreateProjectDto, UpdateProjectDto } from '@n8n/api-types'; export const getAllProjects = async (context: IRestApiContext): Promise => { return await makeRestApiRequest(context, 'GET', '/projects'); }; export const getMyProjects = async (context: IRestApiContext): Promise => { - return await makeRestApiRequest(context, 'GET', '/projects/my-projects', { - includeScopes: true, - }); + return await makeRestApiRequest(context, 'GET', '/projects/my-projects'); }; export const getPersonalProject = async (context: IRestApiContext): Promise => { @@ -28,17 +21,17 @@ export const getProject = async (context: IRestApiContext, id: string): Promise< export const createProject = async ( context: IRestApiContext, - req: ProjectCreateRequest, + payload: CreateProjectDto, ): Promise => { - return await makeRestApiRequest(context, 'POST', '/projects', req); + return await makeRestApiRequest(context, 'POST', '/projects', payload); }; export const updateProject = async ( context: IRestApiContext, - req: ProjectUpdateRequest, + id: Project['id'], + payload: UpdateProjectDto, ): Promise => { - const { id, name, icon, relations } = req; - await makeRestApiRequest(context, 'PATCH', `/projects/${id}`, { name, icon, relations }); + await makeRestApiRequest(context, 'PATCH', `/projects/${id}`, payload); }; export const deleteProject = async ( diff --git a/packages/editor-ui/src/api/schemaPreview.ts b/packages/editor-ui/src/api/schemaPreview.ts new file mode 100644 index 0000000000..9250993929 --- /dev/null +++ b/packages/editor-ui/src/api/schemaPreview.ts @@ -0,0 +1,28 @@ +import { request } from '@/utils/apiUtils'; +import type { JSONSchema7 } from 'json-schema'; + +export type GetSchemaPreviewOptions = { + nodeType: string; + version: number; + resource?: string; + operation?: string; +}; + +const padVersion = (version: number) => { + return version.toString().split('.').concat(['0', '0']).slice(0, 3).join('.'); +}; + +export const getSchemaPreview = async ( + baseUrl: string, + options: GetSchemaPreviewOptions, +): Promise => { + const { nodeType, version, resource, operation } = options; + const versionString = padVersion(version); + const path = ['schemas', nodeType, versionString, resource, operation].filter(Boolean).join('/'); + return await request({ + method: 'GET', + baseURL: baseUrl, + endpoint: `${path}.json`, + withCredentials: false, + }); +}; diff --git a/packages/editor-ui/src/api/sourceControl.ts b/packages/editor-ui/src/api/sourceControl.ts index c8e4eebcbc..4553856928 100644 --- a/packages/editor-ui/src/api/sourceControl.ts +++ b/packages/editor-ui/src/api/sourceControl.ts @@ -1,7 +1,10 @@ -import type { IDataObject } from 'n8n-workflow'; +import type { + PullWorkFolderRequestDto, + PushWorkFolderRequestDto, + SourceControlledFile, +} from '@n8n/api-types'; import type { IRestApiContext } from '@/Interface'; import type { - SourceControlAggregatedFile, SourceControlPreferences, SourceControlStatus, SshKeyTypes, @@ -22,15 +25,15 @@ const createPreferencesRequestFn = export const pushWorkfolder = async ( context: IRestApiContext, - data: IDataObject, + data: PushWorkFolderRequestDto, ): Promise => { return await makeRestApiRequest(context, 'POST', `${sourceControlApiRoot}/push-workfolder`, data); }; export const pullWorkfolder = async ( context: IRestApiContext, - data: IDataObject, -): Promise => { + data: PullWorkFolderRequestDto, +): Promise => { return await makeRestApiRequest(context, 'POST', `${sourceControlApiRoot}/pull-workfolder`, data); }; @@ -60,7 +63,7 @@ export const getAggregatedStatus = async ( preferLocalVersion: boolean; verbose: boolean; } = { direction: 'push', preferLocalVersion: true, verbose: false }, -): Promise => { +): Promise => { return await makeRestApiRequest(context, 'GET', `${sourceControlApiRoot}/get-status`, options); }; diff --git a/packages/editor-ui/src/api/tags.ts b/packages/editor-ui/src/api/tags.ts index 0e429b6433..d85f84cb5d 100644 --- a/packages/editor-ui/src/api/tags.ts +++ b/packages/editor-ui/src/api/tags.ts @@ -1,29 +1,26 @@ import type { IRestApiContext, ITag } from '@/Interface'; import { makeRestApiRequest } from '@/utils/apiUtils'; +import type { CreateOrUpdateTagRequestDto, RetrieveTagQueryDto } from '@n8n/api-types'; type TagsApiEndpoint = '/tags' | '/annotation-tags'; -export interface ITagsApi { - getTags: (context: IRestApiContext, withUsageCount?: boolean) => Promise; - createTag: (context: IRestApiContext, params: { name: string }) => Promise; - updateTag: (context: IRestApiContext, id: string, params: { name: string }) => Promise; - deleteTag: (context: IRestApiContext, id: string) => Promise; -} - -export function createTagsApi(endpoint: TagsApiEndpoint): ITagsApi { +export function createTagsApi(endpoint: TagsApiEndpoint) { return { - getTags: async (context: IRestApiContext, withUsageCount = false): Promise => { - return await makeRestApiRequest(context, 'GET', endpoint, { withUsageCount }); + getTags: async (context: IRestApiContext, data: RetrieveTagQueryDto): Promise => { + return await makeRestApiRequest(context, 'GET', endpoint, data); }, - createTag: async (context: IRestApiContext, params: { name: string }): Promise => { - return await makeRestApiRequest(context, 'POST', endpoint, params); + createTag: async ( + context: IRestApiContext, + data: CreateOrUpdateTagRequestDto, + ): Promise => { + return await makeRestApiRequest(context, 'POST', endpoint, data); }, updateTag: async ( context: IRestApiContext, id: string, - params: { name: string }, + data: CreateOrUpdateTagRequestDto, ): Promise => { - return await makeRestApiRequest(context, 'PATCH', `${endpoint}/${id}`, params); + return await makeRestApiRequest(context, 'PATCH', `${endpoint}/${id}`, data); }, deleteTag: async (context: IRestApiContext, id: string): Promise => { return await makeRestApiRequest(context, 'DELETE', `${endpoint}/${id}`); diff --git a/packages/editor-ui/src/api/testDefinition.ee.ts b/packages/editor-ui/src/api/testDefinition.ee.ts index c515b7a783..52e344d608 100644 --- a/packages/editor-ui/src/api/testDefinition.ee.ts +++ b/packages/editor-ui/src/api/testDefinition.ee.ts @@ -11,7 +11,7 @@ export interface TestDefinitionRecord { updatedAt?: string; createdAt?: string; annotationTag?: string | null; - mockedNodes?: Array<{ name: string }>; + mockedNodes?: Array<{ name: string; id: string }>; } interface CreateTestDefinitionParams { @@ -25,7 +25,7 @@ export interface UpdateTestDefinitionParams { evaluationWorkflowId?: string | null; annotationTagId?: string | null; description?: string | null; - mockedNodes?: Array<{ name: string }>; + mockedNodes?: Array<{ name: string; id: string }>; } export interface UpdateTestResponse { @@ -43,7 +43,7 @@ export interface UpdateTestResponse { export interface TestRunRecord { id: string; testDefinitionId: string; - status: 'new' | 'running' | 'completed' | 'error'; + status: 'new' | 'running' | 'completed' | 'error' | 'cancelled'; metrics?: Record; createdAt: string; updatedAt: string; @@ -61,6 +61,21 @@ interface DeleteTestRunParams { runId: string; } +export interface TestCaseExecutionRecord { + id: string; + testRunId: string; + executionId: string; + pastExecutionId: string; + evaluationExecutionId: string; + status: 'running' | 'completed' | 'error'; + createdAt: string; + updatedAt: string; + runAt: string; + metrics?: Record; + errorCode?: string; + errorDetails?: Record; +} + const endpoint = '/evaluation/test-definitions'; const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) => `${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`; @@ -221,6 +236,21 @@ export const startTestRun = async (context: IRestApiContext, testDefinitionId: s return response as { success: boolean }; }; +export const cancelTestRun = async ( + context: IRestApiContext, + testDefinitionId: string, + testRunId: string, +) => { + const response = await request({ + method: 'POST', + baseURL: context.baseUrl, + endpoint: `${endpoint}/${testDefinitionId}/runs/${testRunId}/cancel`, + headers: { 'push-ref': context.pushRef }, + }); + // CLI is returning the response without wrapping it in `data` key + return response as { success: boolean }; +}; + // Delete a test run export const deleteTestRun = async (context: IRestApiContext, params: DeleteTestRunParams) => { return await makeRestApiRequest<{ success: boolean }>( @@ -229,3 +259,19 @@ export const deleteTestRun = async (context: IRestApiContext, params: DeleteTest getRunsEndpoint(params.testDefinitionId, params.runId), ); }; + +const getRunExecutionsEndpoint = (testDefinitionId: string, runId: string) => + `${endpoint}/${testDefinitionId}/runs/${runId}/cases`; + +// Get all test cases of a test run +export const getTestCaseExecutions = async ( + context: IRestApiContext, + testDefinitionId: string, + runId: string, +) => { + return await makeRestApiRequest( + context, + 'GET', + getRunExecutionsEndpoint(testDefinitionId, runId), + ); +}; diff --git a/packages/editor-ui/src/components/AiStarsIcon.vue b/packages/editor-ui/src/components/AiStarsIcon.vue new file mode 100644 index 0000000000..44958d7feb --- /dev/null +++ b/packages/editor-ui/src/components/AiStarsIcon.vue @@ -0,0 +1,37 @@ + + + diff --git a/packages/editor-ui/src/components/ApiKeyCard.vue b/packages/editor-ui/src/components/ApiKeyCard.vue new file mode 100644 index 0000000000..0c3fe82975 --- /dev/null +++ b/packages/editor-ui/src/components/ApiKeyCard.vue @@ -0,0 +1,125 @@ + + + + + diff --git a/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.test.ts b/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.test.ts new file mode 100644 index 0000000000..3aac4e2434 --- /dev/null +++ b/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.test.ts @@ -0,0 +1,235 @@ +import { createComponentRenderer } from '@/__tests__/render'; +import { createTestingPinia } from '@pinia/testing'; +import { API_KEY_CREATE_OR_EDIT_MODAL_KEY, STORES } from '@/constants'; +import { cleanupAppModals, createAppModals, mockedStore, retry } from '@/__tests__/utils'; +import ApiKeyEditModal from './ApiKeyCreateOrEditModal.vue'; +import { fireEvent } from '@testing-library/vue'; + +import { useApiKeysStore } from '@/stores/apiKeys.store'; +import { DateTime } from 'luxon'; +import type { ApiKeyWithRawValue } from '@n8n/api-types'; + +const renderComponent = createComponentRenderer(ApiKeyEditModal, { + pinia: createTestingPinia({ + initialState: { + [STORES.UI]: { + modalsById: { + [API_KEY_CREATE_OR_EDIT_MODAL_KEY]: { open: true }, + }, + }, + }, + }), +}); + +const testApiKey: ApiKeyWithRawValue = { + id: '123', + label: 'new api key', + apiKey: '123456***', + createdAt: new Date().toString(), + updatedAt: new Date().toString(), + rawApiKey: '123456', + expiresAt: 0, +}; + +const apiKeysStore = mockedStore(useApiKeysStore); + +describe('ApiKeyCreateOrEditModal', () => { + beforeEach(() => { + createAppModals(); + }); + + afterEach(() => { + cleanupAppModals(); + vi.clearAllMocks(); + }); + + test('should allow creating API key with default expiration (30 days)', async () => { + apiKeysStore.createApiKey.mockResolvedValue(testApiKey); + + const { getByText, getByPlaceholderText } = renderComponent({ + props: { + mode: 'new', + }, + }); + + await retry(() => expect(getByText('Create API Key')).toBeInTheDocument()); + expect(getByText('Label')).toBeInTheDocument(); + + const inputLabel = getByPlaceholderText('e.g Internal Project'); + const saveButton = getByText('Save'); + + expect(inputLabel).toBeInTheDocument(); + expect(saveButton).toBeInTheDocument(); + + await fireEvent.update(inputLabel, 'new label'); + + await fireEvent.click(saveButton); + + expect(getByText('API Key Created')).toBeInTheDocument(); + + expect(getByText('Done')).toBeInTheDocument(); + + expect( + getByText('Make sure to copy your API key now as you will not be able to see this again.'), + ).toBeInTheDocument(); + + expect(getByText('You can find more details in')).toBeInTheDocument(); + + expect(getByText('the API documentation')).toBeInTheDocument(); + + expect(getByText('Click to copy')).toBeInTheDocument(); + + expect(getByText('new api key')).toBeInTheDocument(); + }); + + test('should allow creating API key with custom expiration', async () => { + apiKeysStore.createApiKey.mockResolvedValue({ + id: '123', + label: 'new api key', + apiKey: '123456', + createdAt: new Date().toString(), + updatedAt: new Date().toString(), + rawApiKey: '***456', + expiresAt: 0, + }); + + const { getByText, getByPlaceholderText, getByTestId } = renderComponent({ + props: { + mode: 'new', + }, + }); + + await retry(() => expect(getByText('Create API Key')).toBeInTheDocument()); + expect(getByText('Label')).toBeInTheDocument(); + + const inputLabel = getByPlaceholderText('e.g Internal Project'); + const saveButton = getByText('Save'); + const expirationSelect = getByTestId('expiration-select'); + + expect(inputLabel).toBeInTheDocument(); + expect(saveButton).toBeInTheDocument(); + expect(expirationSelect).toBeInTheDocument(); + + await fireEvent.update(inputLabel, 'new label'); + + await fireEvent.click(expirationSelect); + + const customOption = getByText('Custom'); + + expect(customOption).toBeInTheDocument(); + + await fireEvent.click(customOption); + + const customExpirationInput = getByPlaceholderText('yyyy-mm-dd'); + + expect(customExpirationInput).toBeInTheDocument(); + + await fireEvent.input(customExpirationInput, '2029-12-31'); + + await fireEvent.click(saveButton); + + expect(getByText('***456')).toBeInTheDocument(); + + expect(getByText('API Key Created')).toBeInTheDocument(); + + expect(getByText('Done')).toBeInTheDocument(); + + expect( + getByText('Make sure to copy your API key now as you will not be able to see this again.'), + ).toBeInTheDocument(); + + expect(getByText('You can find more details in')).toBeInTheDocument(); + + expect(getByText('the API documentation')).toBeInTheDocument(); + + expect(getByText('Click to copy')).toBeInTheDocument(); + + expect(getByText('new api key')).toBeInTheDocument(); + }); + + test('should allow creating API key with no expiration', async () => { + apiKeysStore.createApiKey.mockResolvedValue(testApiKey); + + const { getByText, getByPlaceholderText, getByTestId } = renderComponent({ + props: { + mode: 'new', + }, + }); + + await retry(() => expect(getByText('Create API Key')).toBeInTheDocument()); + expect(getByText('Label')).toBeInTheDocument(); + + const inputLabel = getByPlaceholderText('e.g Internal Project'); + const saveButton = getByText('Save'); + const expirationSelect = getByTestId('expiration-select'); + + expect(inputLabel).toBeInTheDocument(); + expect(saveButton).toBeInTheDocument(); + expect(expirationSelect).toBeInTheDocument(); + + await fireEvent.update(inputLabel, 'new label'); + + await fireEvent.click(expirationSelect); + + const noExpirationOption = getByText('No Expiration'); + + expect(noExpirationOption).toBeInTheDocument(); + + await fireEvent.click(noExpirationOption); + + await fireEvent.click(saveButton); + + expect(getByText('API Key Created')).toBeInTheDocument(); + + expect(getByText('Done')).toBeInTheDocument(); + + expect( + getByText('Make sure to copy your API key now as you will not be able to see this again.'), + ).toBeInTheDocument(); + + expect(getByText('You can find more details in')).toBeInTheDocument(); + + expect(getByText('the API documentation')).toBeInTheDocument(); + + expect(getByText('Click to copy')).toBeInTheDocument(); + + expect(getByText('new api key')).toBeInTheDocument(); + }); + + test('should allow editing API key label', async () => { + apiKeysStore.apiKeys = [testApiKey]; + + apiKeysStore.updateApiKey.mockResolvedValue(); + + const { getByText, getByTestId } = renderComponent({ + props: { + mode: 'edit', + activeId: '123', + }, + }); + + await retry(() => expect(getByText('Edit API Key')).toBeInTheDocument()); + + expect(getByText('Label')).toBeInTheDocument(); + + const formattedDate = DateTime.fromMillis(Date.parse(testApiKey.createdAt)).toFormat( + 'ccc, MMM d yyyy', + ); + + expect(getByText(`API key was created on ${formattedDate}`)).toBeInTheDocument(); + + const labelInput = getByTestId('api-key-label'); + + expect((labelInput as unknown as HTMLInputElement).value).toBe('new api key'); + + await fireEvent.update(labelInput, 'updated api key'); + + const editButton = getByText('Edit'); + + expect(editButton).toBeInTheDocument(); + + await fireEvent.click(editButton); + + expect(apiKeysStore.updateApiKey).toHaveBeenCalledWith('123', { label: 'updated api key' }); + }); +}); diff --git a/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.vue b/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.vue new file mode 100644 index 0000000000..2ce3eb050a --- /dev/null +++ b/packages/editor-ui/src/components/ApiKeyCreateOrEditModal.vue @@ -0,0 +1,389 @@ + + + + diff --git a/packages/editor-ui/src/components/AssignmentCollection/Assignment.vue b/packages/editor-ui/src/components/AssignmentCollection/Assignment.vue index 614ae3f590..1e9c827234 100644 --- a/packages/editor-ui/src/components/AssignmentCollection/Assignment.vue +++ b/packages/editor-ui/src/components/AssignmentCollection/Assignment.vue @@ -4,14 +4,13 @@ import InputTriple from '@/components/InputTriple/InputTriple.vue'; import ParameterInputFull from '@/components/ParameterInputFull.vue'; import ParameterInputHint from '@/components/ParameterInputHint.vue'; import ParameterIssues from '@/components/ParameterIssues.vue'; -import { useWorkflowHelpers } from '@/composables/useWorkflowHelpers'; -import { useWorkflowsStore } from '@/stores/workflows.store'; -import { isExpression, stringifyExpressionResult } from '@/utils/expressions'; -import type { AssignmentValue, INodeProperties, Result } from 'n8n-workflow'; +import { useResolvedExpression } from '@/composables/useResolvedExpression'; +import useEnvironmentsStore from '@/stores/environments.ee.store'; +import { useNDVStore } from '@/stores/ndv.store'; +import type { AssignmentValue, INodeProperties } from 'n8n-workflow'; import { computed, ref } from 'vue'; import TypeSelect from './TypeSelect.vue'; -import { useNDVStore } from '@/stores/ndv.store'; -import { useRouter } from 'vue-router'; +import { N8nIconButton } from 'n8n-design-system'; interface Props { path: string; @@ -32,8 +31,7 @@ const emit = defineEmits<{ }>(); const ndvStore = useNDVStore(); -const router = useRouter(); -const { resolveExpression } = useWorkflowHelpers({ router }); +const environmentsStore = useEnvironmentsStore(); const assignmentTypeToNodeProperty = ( type: string, @@ -76,48 +74,21 @@ const valueParameter = computed(() => { }; }); -const hint = computed(() => { - const { value } = assignment.value; - if (typeof value !== 'string' || !value.startsWith('=')) { - return ''; - } +const value = computed(() => assignment.value.value); - let result: Result; - try { - const resolvedValue = resolveExpression( - value, - undefined, - ndvStore.isInputParentOfActiveNode - ? { - targetItem: ndvStore.expressionTargetItem ?? undefined, - inputNodeName: ndvStore.ndvInputNodeName, - inputRunIndex: ndvStore.ndvInputRunIndex, - inputBranchIndex: ndvStore.ndvInputBranchIndex, - } - : {}, - ) as unknown; - - result = { ok: true, result: resolvedValue }; - } catch (error) { - result = { ok: false, error }; - } - - const hasRunData = - !!useWorkflowsStore().workflowExecutionData?.data?.resultData?.runData[ - ndvStore.activeNode?.name ?? '' - ]; - - return stringifyExpressionResult(result, hasRunData); +const resolvedAdditionalExpressionData = computed(() => { + return { $vars: environmentsStore.variablesAsObject }; }); +const { resolvedExpressionString, isExpression } = useResolvedExpression({ + expression: value, + additionalData: resolvedAdditionalExpressionData, +}); + +const hint = computed(() => resolvedExpressionString.value); + const highlightHint = computed(() => Boolean(hint.value && ndvStore.getHoveringItem)); -const valueIsExpression = computed(() => { - const { value } = assignment.value; - - return typeof value === 'string' && isExpression(value); -}); - const onAssignmentNameChange = (update: IUpdateInformation): void => { assignment.value.name = update.value as string; }; @@ -125,7 +96,7 @@ const onAssignmentNameChange = (update: IUpdateInformation): void => { const onAssignmentTypeChange = (update: string): void => { assignment.value.type = update; - if (update === 'boolean' && !valueIsExpression.value) { + if (update === 'boolean' && !isExpression.value) { assignment.value.value = false; } }; @@ -160,7 +131,7 @@ const onBlur = (): void => { icon="grip-vertical" :class="[$style.iconButton, $style.defaultTopPadding, 'drag-handle']" > - { data-test-id="assignment-remove" :class="[$style.iconButton, $style.extraTopPadding]" @click="onRemove" - > + >
diff --git a/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.test.ts b/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.test.ts index ba5070e335..5264821232 100644 --- a/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.test.ts +++ b/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { mount } from '@vue/test-utils'; import { createTestingPinia } from '@pinia/testing'; -import ButtonParameter from '@/components/ButtonParameter/ButtonParameter.vue'; +import ButtonParameter, { type Props } from '@/components/ButtonParameter/ButtonParameter.vue'; import { useNDVStore } from '@/stores/ndv.store'; import { useWorkflowsStore } from '@/stores/workflows.store'; import { usePostHog } from '@/stores/posthog.store'; @@ -20,7 +20,7 @@ vi.mock('@/composables/useI18n'); vi.mock('@/composables/useToast'); describe('ButtonParameter', () => { - const defaultProps = { + const defaultProps: Props = { parameter: { name: 'testParam', displayName: 'Test Parameter', @@ -38,6 +38,7 @@ describe('ButtonParameter', () => { }, } as INodeProperties, value: '', + isReadOnly: false, path: 'testPath', }; @@ -78,9 +79,9 @@ describe('ButtonParameter', () => { } as any); }); - const mountComponent = (props = defaultProps) => { + const mountComponent = (props: Partial = {}) => { return mount(ButtonParameter, { - props, + props: { ...defaultProps, ...props }, global: { plugins: [createTestingPinia()], }, @@ -134,4 +135,10 @@ describe('ButtonParameter', () => { expect(useToast().showMessage).toHaveBeenCalled(); }); + + it('disables input and button when in read only mode', async () => { + const wrapper = mountComponent({ isReadOnly: true }); + expect(wrapper.find('textarea').attributes('disabled')).toBeDefined(); + expect(wrapper.find('button').attributes('disabled')).toBeDefined(); + }); }); diff --git a/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.vue b/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.vue index a28b68d3dd..e10933bd19 100644 --- a/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.vue +++ b/packages/editor-ui/src/components/ButtonParameter/ButtonParameter.vue @@ -14,7 +14,6 @@ import { getTextareaCursorPosition, } from './utils'; import { useTelemetry } from '@/composables/useTelemetry'; -import { useUIStore } from '@/stores/ui.store'; import { propertyNameFromExpression } from '../../utils/mappingUtils'; @@ -24,11 +23,13 @@ const emit = defineEmits<{ valueChanged: [value: IUpdateInformation]; }>(); -const props = defineProps<{ +export type Props = { parameter: INodeProperties; value: string; path: string; -}>(); + isReadOnly?: boolean; +}; +const props = defineProps(); const { activeNode } = useNDVStore(); @@ -48,8 +49,7 @@ const buttonLabel = computed( () => props.parameter.typeOptions?.buttonConfig?.label ?? props.parameter.displayName, ); const isSubmitEnabled = computed(() => { - if (!hasExecutionData.value) return false; - if (!prompt.value) return false; + if (!hasExecutionData.value || !prompt.value || props.isReadOnly) return false; const maxlength = inputFieldMaxLength.value; if (maxlength && prompt.value.length > maxlength) return false; @@ -156,16 +156,6 @@ function onPromptInput(inputValue: string) { }); } -function useDarkBackdrop(): string { - const theme = useUIStore().appliedTheme; - - if (theme === 'light') { - return 'background-color: var(--color-background-xlight);'; - } else { - return 'background-color: var(--color-background-light);'; - } -} - onMounted(() => { parentNodes.value = getParentNodes(); }); @@ -213,8 +203,11 @@ async function updateCursorPositionOnMouseMove(event: MouseEvent, activeDrop: bo color="text-dark" > -
-
+
+
diff --git a/packages/editor-ui/src/components/ButtonParameter/utils.ts b/packages/editor-ui/src/components/ButtonParameter/utils.ts index 1044477bbb..47a4081963 100644 --- a/packages/editor-ui/src/components/ButtonParameter/utils.ts +++ b/packages/editor-ui/src/components/ButtonParameter/utils.ts @@ -161,28 +161,24 @@ export function reducePayloadSizeOrThrow( error: Error, averageTokenLength = 4, ) { - try { - let remainingTokensToReduce = calculateRemainingTokens(error); + let remainingTokensToReduce = calculateRemainingTokens(error); - const [remaining, parentNodesTokenCount] = trimParentNodesSchema( - payload, - remainingTokensToReduce, - averageTokenLength, - ); + const [remaining, parentNodesTokenCount] = trimParentNodesSchema( + payload, + remainingTokensToReduce, + averageTokenLength, + ); - remainingTokensToReduce = remaining; + remainingTokensToReduce = remaining; - remainingTokensToReduce = trimInputSchemaProperties( - payload, - remainingTokensToReduce, - averageTokenLength, - parentNodesTokenCount, - ); + remainingTokensToReduce = trimInputSchemaProperties( + payload, + remainingTokensToReduce, + averageTokenLength, + parentNodesTokenCount, + ); - if (remainingTokensToReduce > 0) throw error; - } catch (e) { - throw e; - } + if (remainingTokensToReduce > 0) throw error; } export async function generateCodeForAiTransform(prompt: string, path: string, retries = 1) { diff --git a/packages/editor-ui/src/components/CanvasChat/CanvasChat.test.ts b/packages/editor-ui/src/components/CanvasChat/CanvasChat.test.ts index 47869583a4..153b7eabb9 100644 --- a/packages/editor-ui/src/components/CanvasChat/CanvasChat.test.ts +++ b/packages/editor-ui/src/components/CanvasChat/CanvasChat.test.ts @@ -38,6 +38,13 @@ vi.mock('@/composables/useToast', () => { }, }; }); + +vi.mock('@/stores/pushConnection.store', () => ({ + usePushConnectionStore: vi.fn().mockReturnValue({ + isConnected: true, + }), +})); + // Test data const mockNodes: INodeUi[] = [ { @@ -215,6 +222,7 @@ describe('CanvasChat', () => { // Send message const input = await findByTestId('chat-input'); await userEvent.type(input, 'Hello AI!'); + await userEvent.keyboard('{Enter}'); // Verify message and response @@ -230,28 +238,28 @@ describe('CanvasChat', () => { // Verify workflow execution expect(workflowsStore.runWorkflow).toHaveBeenCalledWith( expect.objectContaining({ - runData: { - 'When chat message received': [ - { - data: { - main: [ - [ - { - json: { - action: 'sendMessage', - chatInput: 'Hello AI!', - sessionId: expect.any(String), - }, + runData: undefined, + triggerToStartFrom: { + name: 'When chat message received', + data: { + data: { + main: [ + [ + { + json: { + action: 'sendMessage', + chatInput: 'Hello AI!', + sessionId: expect.any(String), }, - ], + }, ], - }, - executionStatus: 'success', - executionTime: 0, - source: [null], - startTime: expect.any(Number), + ], }, - ], + executionStatus: 'success', + executionTime: 0, + source: [null], + startTime: expect.any(Number), + }, }, }), ); @@ -263,14 +271,21 @@ describe('CanvasChat', () => { // Send message const input = await findByTestId('chat-input'); await userEvent.type(input, 'Test message'); + + // Since runWorkflow resolve is mocked, the isWorkflowRunning will be false from the first run. + // This means that the loading state never gets a chance to appear. + // We're forcing isWorkflowRunning to be true for the first run. + workflowsStore.isWorkflowRunning = true; await userEvent.keyboard('{Enter}'); await waitFor(() => expect(queryByTestId('chat-message-typing')).toBeInTheDocument()); + workflowsStore.isWorkflowRunning = false; workflowsStore.getWorkflowExecution = { ...(mockWorkflowExecution as unknown as IExecutionResponse), status: 'success', }; + await waitFor(() => expect(queryByTestId('chat-message-typing')).not.toBeInTheDocument()); }); diff --git a/packages/editor-ui/src/components/CanvasChat/CanvasChat.vue b/packages/editor-ui/src/components/CanvasChat/CanvasChat.vue index adfbc8cdd4..97505a0b40 100644 --- a/packages/editor-ui/src/components/CanvasChat/CanvasChat.vue +++ b/packages/editor-ui/src/components/CanvasChat/CanvasChat.vue @@ -175,47 +175,40 @@ const closePanel = () => { // This function creates a promise that resolves when the workflow execution completes // It's used to handle the loading state while waiting for the workflow to finish async function createExecutionPromise() { - let resolvePromise: () => void; - const promise = new Promise((resolve) => { - resolvePromise = resolve; - }); - - // Watch for changes in the workflow execution status - const stopWatch = watch( - () => workflowsStore.getWorkflowExecution?.status, - (newStatus) => { - // If the status is no longer 'running', resolve the promise - if (newStatus && newStatus !== 'running') { - resolvePromise(); - // Stop the watcher when the promise is resolved - stopWatch(); + return await new Promise((resolve) => { + const resolveIfFinished = (isRunning: boolean) => { + if (!isRunning) { + unwatch(); + resolve(); } - }, - { immediate: true }, // Check the status immediately when the watcher is set up - ); + }; - // Return the promise, which will resolve when the workflow execution is complete - // This allows the caller to await the execution and handle the loading state appropriately - return await promise; + // Watch for changes in the workflow execution status + const unwatch = watch(() => workflowsStore.isWorkflowRunning, resolveIfFinished); + resolveIfFinished(workflowsStore.isWorkflowRunning); + }); } async function onRunChatWorkflow(payload: RunWorkflowChatPayload) { - try { - const response = await runWorkflow({ - triggerNode: payload.triggerNode, - nodeData: payload.nodeData, - source: payload.source, - }); + const runWorkflowOptions: Parameters[0] = { + triggerNode: payload.triggerNode, + nodeData: payload.nodeData, + source: payload.source, + }; - if (response) { - await createExecutionPromise(); - workflowsStore.appendChatMessage(payload.message); - return response; - } - return; - } catch (error) { - throw error; + if (workflowsStore.chatPartialExecutionDestinationNode) { + runWorkflowOptions.destinationNode = workflowsStore.chatPartialExecutionDestinationNode; + workflowsStore.chatPartialExecutionDestinationNode = null; } + + const response = await runWorkflow(runWorkflowOptions); + + if (response) { + await createExecutionPromise(); + workflowsStore.appendChatMessage(payload.message); + return response; + } + return; } // Initialize chat config diff --git a/packages/editor-ui/src/components/CanvasControls.vue b/packages/editor-ui/src/components/CanvasControls.vue index 223c1cc8de..e0bced9be0 100644 --- a/packages/editor-ui/src/components/CanvasControls.vue +++ b/packages/editor-ui/src/components/CanvasControls.vue @@ -3,7 +3,7 @@ import { onBeforeMount, onBeforeUnmount } from 'vue'; import { storeToRefs } from 'pinia'; import { useCanvasStore } from '@/stores/canvas.store'; import KeyboardShortcutTooltip from '@/components/KeyboardShortcutTooltip.vue'; -import { useDeviceSupport } from 'n8n-design-system'; +import { useDeviceSupport } from '@n8n/composables/useDeviceSupport'; import { useI18n } from '@/composables/useI18n'; const canvasStore = useCanvasStore(); diff --git a/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue b/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue index c17ff804c0..74bbddacc3 100644 --- a/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue +++ b/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue @@ -37,7 +37,7 @@ const props = withDefaults(defineProps(), { language: 'javaScript', isReadOnly: false, rows: 4, - id: crypto.randomUUID(), + id: () => crypto.randomUUID(), }); const emit = defineEmits<{ 'update:modelValue': [value: string]; diff --git a/packages/editor-ui/src/components/ContextMenu/ContextMenu.vue b/packages/editor-ui/src/components/ContextMenu/ContextMenu.vue index 0fd54b0a73..2425559e83 100644 --- a/packages/editor-ui/src/components/ContextMenu/ContextMenu.vue +++ b/packages/editor-ui/src/components/ContextMenu/ContextMenu.vue @@ -29,7 +29,9 @@ function onActionSelect(item: string) { } function closeMenu(event: MouseEvent) { - event.preventDefault(); + if (event.cancelable) { + event.preventDefault(); + } event.stopPropagation(); contextMenu.close(); } diff --git a/packages/editor-ui/src/components/CredentialCard.vue b/packages/editor-ui/src/components/CredentialCard.vue index db40f0d274..4d4dd131f6 100644 --- a/packages/editor-ui/src/components/CredentialCard.vue +++ b/packages/editor-ui/src/components/CredentialCard.vue @@ -29,6 +29,7 @@ const props = withDefaults( defineProps<{ data: ICredentialsResponse; readOnly?: boolean; + needsSetup?: boolean; }>(), { data: () => ({ @@ -146,6 +147,9 @@ function moveResource() { {{ locale.baseText('credentials.item.readonly') }} + + {{ locale.baseText('credentials.item.needsSetup') }} +
@@ -195,10 +199,6 @@ function moveResource() { .cardHeading { font-size: var(--font-size-s); padding: var(--spacing-s) 0 0; - - span { - color: var(--color-text-light); - } } .cardDescription { diff --git a/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.constants.ts b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.constants.ts new file mode 100644 index 0000000000..b23df04547 --- /dev/null +++ b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.constants.ts @@ -0,0 +1,276 @@ +import type { ICredentialMap, ICredentialTypeMap } from '@/Interface'; + +export const TEST_CREDENTIALS: ICredentialMap = { + // OpenAI credential in personal + 1: { + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + id: '1', + name: 'OpenAi account', + data: 'test123', + type: 'openAiApi', + isManaged: false, + homeProject: { + id: '1', + type: 'personal', + name: 'Kobi Dog ', + icon: null, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + sharedWithProjects: [ + { + id: '2', + type: 'team', + name: 'Test Project', + icon: { type: 'icon', value: 'exchange-alt' }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + ], + scopes: [ + 'credential:create', + 'credential:delete', + 'credential:list', + 'credential:move', + 'credential:read', + 'credential:share', + 'credential:update', + ], + }, + // Supabase credential in another project + 2: { + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + id: '2', + name: 'Supabase account', + data: 'test123', + type: 'supabaseApi', + isManaged: false, + homeProject: { + id: '2', + type: 'team', + name: 'Test Project', + icon: { type: 'icon', value: 'exchange-alt' }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + sharedWithProjects: [], + scopes: [ + 'credential:create', + 'credential:delete', + 'credential:list', + 'credential:move', + 'credential:read', + 'credential:share', + 'credential:update', + ], + }, + // Slack account in personal + 3: { + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + id: '3', + name: 'Slack account', + data: 'test123', + type: 'slackOAuth2Api', + isManaged: false, + homeProject: { + id: '1', + type: 'personal', + name: 'Kobi Dog ', + icon: null, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + sharedWithProjects: [], + scopes: [ + 'credential:create', + 'credential:delete', + 'credential:list', + 'credential:move', + 'credential:read', + 'credential:share', + 'credential:update', + ], + }, + // OpenAI credential in another project + 4: { + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + id: '4', + name: '[PROJECT] OpenAI Account', + data: 'test123', + type: 'openAiApi', + isManaged: false, + homeProject: { + id: '2', + type: 'team', + name: 'Test Project', + icon: { type: 'icon', value: 'exchange-alt' }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + sharedWithProjects: [], + scopes: [ + 'credential:create', + 'credential:delete', + 'credential:list', + 'credential:move', + 'credential:read', + 'credential:share', + 'credential:update', + ], + }, +}; + +export const TEST_CREDENTIAL_TYPES: ICredentialTypeMap = { + openAiApi: { + name: 'openAiApi', + displayName: 'OpenAi', + documentationUrl: 'openAi', + properties: [ + { + displayName: 'API Key', + name: 'apiKey', + type: 'string', + typeOptions: { + password: true, + }, + required: true, + default: '', + }, + { + displayName: 'Base URL', + name: 'url', + type: 'string', + default: 'https://api.openai.com/v1', + description: 'Override the base URL for the API', + }, + ], + authenticate: { + type: 'generic', + properties: { + headers: { + Authorization: '=Bearer {{$credentials.apiKey}}', + }, + }, + }, + test: { + request: { + baseURL: '={{$credentials?.url}}', + url: '/models', + }, + }, + iconUrl: { + light: 'icons/n8n-nodes-base/dist/nodes/OpenAi/openai.svg', + dark: 'icons/n8n-nodes-base/dist/nodes/OpenAi/openai.dark.svg', + }, + supportedNodes: [ + 'n8n-nodes-base.openAi', + '@n8n/n8n-nodes-langchain.embeddingsOpenAi', + '@n8n/n8n-nodes-langchain.lmChatOpenAi', + '@n8n/n8n-nodes-langchain.lmOpenAi', + ], + }, + supabaseApi: { + name: 'supabaseApi', + displayName: 'Supabase API', + documentationUrl: 'supabase', + properties: [ + { + displayName: 'Host', + name: 'host', + type: 'string', + placeholder: 'https://your_account.supabase.co', + default: '', + }, + { + displayName: 'Service Role Secret', + name: 'serviceRole', + type: 'string', + default: '', + typeOptions: { + password: true, + }, + }, + ], + authenticate: { + type: 'generic', + properties: { + headers: { + apikey: '={{$credentials.serviceRole}}', + Authorization: '=Bearer {{$credentials.serviceRole}}', + }, + }, + }, + test: { + request: { + baseURL: '={{$credentials.host}}/rest/v1', + headers: { + Prefer: 'return=representation', + }, + url: '/', + }, + }, + iconUrl: 'icons/n8n-nodes-base/dist/nodes/Supabase/supabase.svg', + supportedNodes: ['n8n-nodes-base.supabase'], + }, + slackOAuth2Api: { + name: 'slackOAuth2Api', + extends: ['oAuth2Api'], + displayName: 'Slack OAuth2 API', + documentationUrl: 'slack', + properties: [ + { + displayName: 'Grant Type', + name: 'grantType', + type: 'hidden', + default: 'authorizationCode', + }, + { + displayName: 'Authorization URL', + name: 'authUrl', + type: 'hidden', + default: 'https://slack.com/oauth/v2/authorize', + }, + { + displayName: 'Access Token URL', + name: 'accessTokenUrl', + type: 'hidden', + default: 'https://slack.com/api/oauth.v2.access', + }, + { + displayName: 'Scope', + name: 'scope', + type: 'hidden', + default: 'chat:write', + }, + { + displayName: 'Auth URI Query Parameters', + name: 'authQueryParameters', + type: 'hidden', + default: + 'user_scope=channels:read channels:write chat:write files:read files:write groups:read im:read mpim:read reactions:read reactions:write stars:read stars:write usergroups:write usergroups:read users.profile:read users.profile:write users:read', + }, + { + displayName: 'Authentication', + name: 'authentication', + type: 'hidden', + default: 'body', + }, + { + displayName: + 'If you get an Invalid Scopes error, make sure you add the correct one here to your Slack integration', + name: 'notice', + type: 'notice', + default: '', + }, + ], + iconUrl: 'icons/n8n-nodes-base/dist/nodes/Slack/slack.svg', + supportedNodes: ['n8n-nodes-base.slack'], + }, +}; + +export const PERSONAL_OPENAI_CREDENTIAL = TEST_CREDENTIALS[1]; +export const PROJECT_OPENAI_CREDENTIAL = TEST_CREDENTIALS[4]; diff --git a/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.ts b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.ts new file mode 100644 index 0000000000..816b78e818 --- /dev/null +++ b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.test.ts @@ -0,0 +1,78 @@ +import { createComponentRenderer } from '@/__tests__/render'; +import { mockedStore } from '@/__tests__/utils'; +import { useCredentialsStore } from '@/stores/credentials.store'; +import { createTestingPinia } from '@pinia/testing'; +import CredentialPicker from './CredentialPicker.vue'; +import { + PERSONAL_OPENAI_CREDENTIAL, + PROJECT_OPENAI_CREDENTIAL, + TEST_CREDENTIAL_TYPES, + TEST_CREDENTIALS, +} from './CredentialPicker.test.constants'; +import userEvent from '@testing-library/user-event'; +import { screen } from '@testing-library/vue'; + +vi.mock('vue-router', () => { + const push = vi.fn(); + const resolve = vi.fn().mockReturnValue({ href: 'https://test.com' }); + return { + useRouter: () => ({ + push, + resolve, + }), + useRoute: () => ({}), + RouterLink: vi.fn(), + }; +}); + +let credentialsStore: ReturnType>; + +const renderComponent = createComponentRenderer(CredentialPicker); + +describe('CredentialPicker', () => { + beforeEach(() => { + createTestingPinia(); + credentialsStore = mockedStore(useCredentialsStore); + credentialsStore.state.credentials = TEST_CREDENTIALS; + credentialsStore.state.credentialTypes = TEST_CREDENTIAL_TYPES; + }); + + it('should render', () => { + expect(() => + renderComponent({ + props: { + appName: 'OpenAI', + credentialType: 'openAiApi', + selectedCredentialId: null, + }, + }), + ).not.toThrowError(); + }); + + it('should only render personal credentials of the specified type', async () => { + const TEST_APP_NAME = 'OpenAI'; + const TEST_CREDENTIAL_TYPE = 'openAiApi'; + const { getByTestId } = renderComponent({ + props: { + appName: TEST_APP_NAME, + credentialType: TEST_CREDENTIAL_TYPE, + selectedCredentialId: null, + }, + }); + expect(getByTestId('credential-dropdown')).toBeInTheDocument(); + expect(getByTestId('credential-dropdown')).toHaveAttribute( + 'credential-type', + TEST_CREDENTIAL_TYPE, + ); + // Open the dropdown + await userEvent.click(getByTestId('credential-dropdown')); + // Personal openAI credential should be in the dropdown + expect( + screen.getByTestId(`node-credentials-select-item-${PERSONAL_OPENAI_CREDENTIAL.id}`), + ).toBeInTheDocument(); + // OpenAI credential that belong to other project should not be in the dropdown + expect( + screen.queryByTestId(`node-credentials-select-item-${PROJECT_OPENAI_CREDENTIAL.id}`), + ).not.toBeInTheDocument(); + }); +}); diff --git a/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.vue b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.vue index 24fce63090..8bddeddd85 100644 --- a/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.vue +++ b/packages/editor-ui/src/components/CredentialPicker/CredentialPicker.vue @@ -26,7 +26,10 @@ const i18n = useI18n(); const wasModalOpenedFromHere = ref(false); const availableCredentials = computed(() => { - return credentialsStore.getCredentialsByType(props.credentialType); + const credByType = credentialsStore.getCredentialsByType(props.credentialType); + // Only show personal credentials since templates are created in personal by default + // Here, we don't care about sharing because credentials cannot be shared with personal project + return credByType.filter((credential) => credential.homeProject?.type === 'personal'); }); const credentialOptions = computed(() => { @@ -98,6 +101,7 @@ listenForModalChanges({ :credential-type="props.credentialType" :credential-options="credentialOptions" :selected-credential-id="props.selectedCredentialId" + data-test-id="credential-dropdown" @credential-selected="onCredentialSelected" @new-credential="createNewCredential" /> diff --git a/packages/editor-ui/src/components/ExpressionParameterInput.vue b/packages/editor-ui/src/components/ExpressionParameterInput.vue index 8706e73991..4217bc12ee 100644 --- a/packages/editor-ui/src/components/ExpressionParameterInput.vue +++ b/packages/editor-ui/src/components/ExpressionParameterInput.vue @@ -55,6 +55,12 @@ const workflowsStore = useWorkflowsStore(); const isDragging = computed(() => ndvStore.isDraggableDragging); +function select() { + if (inlineInput.value) { + inlineInput.value.selectAll(); + } +} + function focus() { if (inlineInput.value) { inlineInput.value.focus(); @@ -162,7 +168,7 @@ watch(isDragging, (newIsDragging) => { onClickOutside(container, (event) => onBlur(event)); -defineExpose({ focus }); +defineExpose({ focus, select }); @@ -1532,6 +1629,13 @@ onUpdated(async () => { align-items: center; } +.parameter-switch { + display: inline-flex; + align-self: flex-start; + justify-items: center; + gap: var(--spacing-xs); +} + .parameter-input { display: inline-block; position: relative; @@ -1706,4 +1810,26 @@ onUpdated(async () => { border-bottom-left-radius: 4px; border-bottom-right-radius: 4px; } + +.noRightCornersInput > * { + --input-border-bottom-right-radius: 0; + --input-border-top-right-radius: 0; +} + +.overrideButton { + align-self: start; +} + +.overrideButtonStandalone { + position: relative; + /* This is to balance for the extra margin on the switch */ + top: -2px; +} + +.overrideButtonInline { + > button { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + } +} diff --git a/packages/editor-ui/src/components/ParameterInputFull.test.ts b/packages/editor-ui/src/components/ParameterInputFull.test.ts new file mode 100644 index 0000000000..9dfe37fb4f --- /dev/null +++ b/packages/editor-ui/src/components/ParameterInputFull.test.ts @@ -0,0 +1,158 @@ +import { renderComponent } from '@/__tests__/render'; +import type { useNDVStore } from '@/stores/ndv.store'; +import { createTestingPinia } from '@pinia/testing'; +import type { useNodeTypesStore } from '@/stores/nodeTypes.store'; +import type { useSettingsStore } from '@/stores/settings.store'; +import { cleanupAppModals, createAppModals } from '@/__tests__/utils'; +import ParameterInputFull from './ParameterInputFull.vue'; +import { FROM_AI_AUTO_GENERATED_MARKER } from 'n8n-workflow'; + +type Writeable = { -readonly [P in keyof T]: T[P] }; + +let mockNdvState: Partial>; +let mockNodeTypesState: Writeable>>; +let mockSettingsState: Writeable>>; + +beforeEach(() => { + mockNdvState = { + hasInputData: true, + activeNode: { + id: '123', + name: 'myParam', + parameters: {}, + position: [0, 0], + type: 'test', + typeVersion: 1, + }, + isInputPanelEmpty: false, + isOutputPanelEmpty: false, + }; + mockNodeTypesState = { + allNodeTypes: [], + }; + mockSettingsState = { + settings: { + releaseChannel: 'stable', + } as never, + isEnterpriseFeatureEnabled: { externalSecrets: false } as never, + }; + createAppModals(); +}); + +vi.mock('@/stores/ndv.store', () => { + return { + useNDVStore: vi.fn(() => mockNdvState), + }; +}); + +vi.mock('@/stores/nodeTypes.store', () => { + return { + useNodeTypesStore: vi.fn(() => mockNodeTypesState), + }; +}); + +vi.mock('@/stores/settings.store', () => { + return { + useSettingsStore: vi.fn(() => mockSettingsState), + }; +}); + +describe('ParameterInputFull.vue', () => { + beforeEach(() => { + vi.clearAllMocks(); + createAppModals(); + }); + + afterEach(() => { + cleanupAppModals(); + }); + + it('should render basic parameter', async () => { + mockNodeTypesState.getNodeType = vi.fn().mockReturnValue({}); + const { getByTestId } = renderComponent(ParameterInputFull, { + pinia: createTestingPinia(), + props: { + path: 'myParam', + parameter: { + displayName: 'My Param', + name: 'myParam', + type: 'string', + }, + }, + }); + expect(getByTestId('parameter-input')).toBeInTheDocument(); + }); + + it('should render parameter with override button inline', async () => { + mockNodeTypesState.getNodeType = vi.fn().mockReturnValue({ + codex: { + categories: ['AI'], + subcategories: { AI: ['Tools'] }, + }, + }); + const { getByTestId } = renderComponent(ParameterInputFull, { + pinia: createTestingPinia(), + props: { + path: 'myParam', + parameter: { + displayName: 'My Param', + name: 'myParam', + type: 'string', + }, + value: '', + }, + }); + expect(getByTestId('parameter-input')).toBeInTheDocument(); + expect(getByTestId('from-ai-override-button')).toBeInTheDocument(); + }); + + it('should render parameter with override button in options', async () => { + mockNodeTypesState.getNodeType = vi.fn().mockReturnValue({ + codex: { + categories: ['AI'], + subcategories: { AI: ['Tools'] }, + }, + }); + const { getByTestId } = renderComponent(ParameterInputFull, { + pinia: createTestingPinia(), + props: { + path: 'myParam', + parameter: { + displayName: 'My Param', + name: 'myParam', + type: 'string', + }, + value: `={{ + 'and the air is free' + + + }}`, + }, + }); + expect(getByTestId('parameter-input')).toBeInTheDocument(); + expect(getByTestId('from-ai-override-button')).toBeInTheDocument(); + }); + + it('should render parameter with active override', async () => { + mockNodeTypesState.getNodeType = vi.fn().mockReturnValue({ + codex: { + categories: ['AI'], + subcategories: { AI: ['Tools'] }, + }, + }); + const { queryByTestId, getByTestId } = renderComponent(ParameterInputFull, { + pinia: createTestingPinia(), + props: { + path: 'myParam', + value: `={{ ${FROM_AI_AUTO_GENERATED_MARKER} $fromAI('myParam') }}`, + parameter: { + displayName: 'My Param', + name: 'myParam', + type: 'string', + }, + }, + }); + expect(getByTestId('fromAI-override-field')).toBeInTheDocument(); + expect(queryByTestId('override-button')).not.toBeInTheDocument(); + }); +}); diff --git a/packages/editor-ui/src/components/ParameterInputFull.vue b/packages/editor-ui/src/components/ParameterInputFull.vue index 332011b9ea..8e2453becf 100644 --- a/packages/editor-ui/src/components/ParameterInputFull.vue +++ b/packages/editor-ui/src/components/ParameterInputFull.vue @@ -1,10 +1,12 @@