From 90d6f0020cf8937edbed9315ff9db15b10f677e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Tue, 11 Feb 2025 17:51:50 +0100 Subject: [PATCH] refactor(core): Move more code out of NodeExecutionFunctions, and add additional tests (no-changelog) (#13195) --- .../__tests__/node-execute-functions.test.ts | 314 --- .../credentials-test-context.ts | 5 +- .../node-execution-context/execute-context.ts | 16 +- .../execute-single-context.ts | 5 +- .../node-execution-context/hook-context.ts | 9 +- .../node-execution-context/index.ts | 8 +- .../load-options-context.ts | 5 +- .../node-execution-context/poll-context.ts | 10 +- .../supply-data-context.ts | 17 +- .../node-execution-context/trigger-context.ts | 12 +- .../construct-execution-metadata.test.ts | 44 + .../utils/__tests__/copy-input-items.test.ts | 49 + .../utils/__tests__/normalize-items.test.ts | 110 ++ .../__tests__/parse-request-object.test.ts | 133 -- .../request-helper-functions.test.ts | 865 ++++++++ .../utils/__tests__/return-json-array.test.ts | 34 + .../scheduling-helper-functions.test.ts | 31 + .../ssh-tunnel-helper-functions.test.ts | 25 + .../webhook-helper-functions.test.ts | 150 ++ .../utils/construct-execution-metadata.ts | 17 + .../utils/copy-input-items.ts | 20 + .../utils/get-input-connection-data.ts | 4 +- .../utils/normalize-items.ts | 47 + .../utils/parse-request-object.ts | 468 ----- .../utils/request-helper-functions.ts | 1743 +++++++++++++++++ .../utils/return-json-array.ts | 25 + .../utils/scheduling-helper-functions.ts | 12 + .../utils/ssh-tunnel-helper-functions.ts | 11 + .../utils/webhook-helper-functions.ts | 67 + .../node-execution-context/webhook-context.ts | 10 +- packages/core/src/node-execute-functions.ts | 1529 +-------------- .../Google/Drive/test/v2/node/helpers.ts | 4 +- 32 files changed, 3291 insertions(+), 2508 deletions(-) delete mode 100644 packages/core/src/__tests__/node-execute-functions.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/construct-execution-metadata.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/copy-input-items.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/normalize-items.test.ts delete mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/return-json-array.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/scheduling-helper-functions.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/ssh-tunnel-helper-functions.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/__tests__/webhook-helper-functions.test.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/construct-execution-metadata.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/copy-input-items.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/normalize-items.ts delete mode 100644 packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/return-json-array.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/scheduling-helper-functions.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/ssh-tunnel-helper-functions.ts create mode 100644 packages/core/src/execution-engine/node-execution-context/utils/webhook-helper-functions.ts diff --git a/packages/core/src/__tests__/node-execute-functions.test.ts b/packages/core/src/__tests__/node-execute-functions.test.ts deleted file mode 100644 index 96306ac90b..0000000000 --- a/packages/core/src/__tests__/node-execute-functions.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { mock } from 'jest-mock-extended'; -import type { - IHttpRequestMethods, - IHttpRequestOptions, - INode, - IRequestOptions, - IWorkflowExecuteAdditionalData, - Workflow, -} from 'n8n-workflow'; -import nock from 'nock'; - -import type { ExecutionLifecycleHooks } from '@/execution-engine/execution-lifecycle-hooks'; -import { - copyInputItems, - invokeAxios, - proxyRequestToAxios, - removeEmptyBody, -} from '@/node-execute-functions'; - -describe('NodeExecuteFunctions', () => { - describe('proxyRequestToAxios', () => { - const baseUrl = 'http://example.de'; - const workflow = mock(); - const hooks = mock(); - const additionalData = mock({ hooks }); - const node = mock(); - - beforeEach(() => { - hooks.runHook.mockClear(); - }); - - test('should rethrow an error with `status` property', async () => { - nock(baseUrl).get('/test').reply(400); - - try { - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - } catch (error) { - expect(error.status).toEqual(400); - } - }); - - test('should not throw if the response status is 200', async () => { - nock(baseUrl).get('/test').reply(200); - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); - }); - - test('should throw if the response status is 403', async () => { - const headers = { 'content-type': 'text/plain' }; - nock(baseUrl).get('/test').reply(403, 'Forbidden', headers); - try { - await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); - } catch (error) { - expect(error.statusCode).toEqual(403); - expect(error.request).toBeUndefined(); - expect(error.response).toMatchObject({ headers, status: 403 }); - expect(error.options).toMatchObject({ - headers: { Accept: '*/*' }, - method: 'get', - url: 'http://example.de/test', - }); - expect(error.config).toBeUndefined(); - expect(error.message).toEqual('403 - "Forbidden"'); - } - expect(hooks.runHook).not.toHaveBeenCalled(); - }); - - test('should not throw if the response status is 404, but `simple` option is set to `false`', async () => { - nock(baseUrl).get('/test').reply(404, 'Not Found'); - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/test`, - simple: false, - }); - - expect(response).toEqual('Not Found'); - expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); - }); - - test('should return full response when `resolveWithFullResponse` is set to true', async () => { - nock(baseUrl).get('/test').reply(404, 'Not Found'); - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/test`, - resolveWithFullResponse: true, - simple: false, - }); - - expect(response).toMatchObject({ - body: 'Not Found', - headers: {}, - statusCode: 404, - statusMessage: 'Not Found', - }); - expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); - }); - - describe('redirects', () => { - test('should forward authorization header', async () => { - nock(baseUrl).get('/redirect').reply(301, '', { Location: 'https://otherdomain.com/test' }); - nock('https://otherdomain.com') - .get('/test') - .reply(200, function () { - return this.req.headers; - }); - - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - auth: { - username: 'testuser', - password: 'testpassword', - }, - headers: { - 'X-Other-Header': 'otherHeaderContent', - }, - resolveWithFullResponse: true, - }); - - expect(response.statusCode).toBe(200); - const forwardedHeaders = JSON.parse(response.body); - expect(forwardedHeaders.authorization).toBe('Basic dGVzdHVzZXI6dGVzdHBhc3N3b3Jk'); - expect(forwardedHeaders['x-other-header']).toBe('otherHeaderContent'); - }); - - test('should follow redirects by default', async () => { - nock(baseUrl) - .get('/redirect') - .reply(301, '', { Location: `${baseUrl}/test` }); - nock(baseUrl).get('/test').reply(200, 'Redirected'); - - const response = await proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - resolveWithFullResponse: true, - }); - - expect(response).toMatchObject({ - body: 'Redirected', - headers: {}, - statusCode: 200, - }); - }); - - test('should not follow redirects when configured', async () => { - nock(baseUrl) - .get('/redirect') - .reply(301, '', { Location: `${baseUrl}/test` }); - nock(baseUrl).get('/test').reply(200, 'Redirected'); - - await expect( - proxyRequestToAxios(workflow, additionalData, node, { - url: `${baseUrl}/redirect`, - resolveWithFullResponse: true, - followRedirect: false, - }), - ).rejects.toThrowError(expect.objectContaining({ statusCode: 301 })); - }); - }); - }); - - describe('invokeAxios', () => { - const baseUrl = 'http://example.de'; - - beforeEach(() => { - nock.cleanAll(); - jest.clearAllMocks(); - }); - - it('should throw error for non-401 status codes', async () => { - nock(baseUrl).get('/test').reply(500, {}); - - await expect(invokeAxios({ url: `${baseUrl}/test` })).rejects.toThrow( - 'Request failed with status code 500', - ); - }); - - it('should throw error on 401 without digest auth challenge', async () => { - nock(baseUrl).get('/test').reply(401, {}); - - await expect( - invokeAxios( - { - url: `${baseUrl}/test`, - }, - { sendImmediately: false }, - ), - ).rejects.toThrow('Request failed with status code 401'); - }); - - it('should make successful requests', async () => { - nock(baseUrl).get('/test').reply(200, { success: true }); - - const response = await invokeAxios({ - url: `${baseUrl}/test`, - }); - - expect(response.status).toBe(200); - expect(response.data).toEqual({ success: true }); - }); - - it('should handle digest auth when receiving 401 with nonce', async () => { - nock(baseUrl) - .get('/test') - .matchHeader('authorization', 'Basic dXNlcjpwYXNz') - .once() - .reply(401, {}, { 'www-authenticate': 'Digest realm="test", nonce="abc123", qop="auth"' }); - - nock(baseUrl) - .get('/test') - .matchHeader( - 'authorization', - /^Digest username="user",realm="test",nonce="abc123",uri="\/test",qop="auth",algorithm="MD5",response="[0-9a-f]{32}"/, - ) - .reply(200, { success: true }); - - const response = await invokeAxios( - { - url: `${baseUrl}/test`, - auth: { - username: 'user', - password: 'pass', - }, - }, - { sendImmediately: false }, - ); - - expect(response.status).toBe(200); - expect(response.data).toEqual({ success: true }); - }); - }); - - describe('copyInputItems', () => { - it('should pick only selected properties', () => { - const output = copyInputItems( - [ - { - json: { - a: 1, - b: true, - c: {}, - }, - }, - ], - ['a'], - ); - expect(output).toEqual([{ a: 1 }]); - }); - - it('should convert undefined to null', () => { - const output = copyInputItems( - [ - { - json: { - a: undefined, - }, - }, - ], - ['a'], - ); - expect(output).toEqual([{ a: null }]); - }); - - it('should clone objects', () => { - const input = { - a: { b: 5 }, - }; - const output = copyInputItems( - [ - { - json: input, - }, - ], - ['a'], - ); - expect(output[0].a).toEqual(input.a); - expect(output[0].a === input.a).toEqual(false); - }); - }); - - describe('removeEmptyBody', () => { - test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( - 'Should remove empty body for %s', - async (method) => { - const requestOptions = { - method, - body: {}, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual(undefined); - }, - ); - - test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( - 'Should not remove non-empty body for %s', - async (method) => { - const requestOptions = { - method, - body: { test: true }, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual({ test: true }); - }, - ); - - test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'Should not remove empty body for %s', - async (method) => { - const requestOptions = { - method, - body: {}, - } as IHttpRequestOptions | IRequestOptions; - removeEmptyBody(requestOptions); - expect(requestOptions.body).toEqual({}); - }, - ); - }); -}); diff --git a/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts b/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts index 9aa99683da..1899b3a443 100644 --- a/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/credentials-test-context.ts @@ -3,8 +3,9 @@ import type { ICredentialTestFunctions } from 'n8n-workflow'; import { Memoized } from '@/decorators'; import { Logger } from '@/logging'; -// eslint-disable-next-line import/no-cycle -import { getSSHTunnelFunctions, proxyRequestToAxios } from '@/node-execute-functions'; + +import { proxyRequestToAxios } from './utils/request-helper-functions'; +import { getSSHTunnelFunctions } from './utils/ssh-tunnel-helper-functions'; export class CredentialTestContext implements ICredentialTestFunctions { readonly helpers: ICredentialTestFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/execute-context.ts b/packages/core/src/execution-engine/node-execution-context/execute-context.ts index ae0ebbd3fb..47d679bc60 100644 --- a/packages/core/src/execution-engine/node-execution-context/execute-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/execute-context.ts @@ -23,16 +23,6 @@ import { NodeConnectionType, } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - returnJsonArray, - copyInputItems, - normalizeItems, - constructExecutionMetaData, - getRequestHelperFunctions, - getSSHTunnelFunctions, -} from '@/node-execute-functions'; - import { BaseExecuteContext } from './base-execute-context'; import { assertBinaryData, @@ -41,9 +31,15 @@ import { getBinaryHelperFunctions, detectBinaryEncoding, } from './utils/binary-helper-functions'; +import { constructExecutionMetaData } from './utils/construct-execution-metadata'; +import { copyInputItems } from './utils/copy-input-items'; import { getDeduplicationHelperFunctions } from './utils/deduplication-helper-functions'; import { getFileSystemHelperFunctions } from './utils/file-system-helper-functions'; import { getInputConnectionData } from './utils/get-input-connection-data'; +import { normalizeItems } from './utils/normalize-items'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; +import { getSSHTunnelFunctions } from './utils/ssh-tunnel-helper-functions'; export class ExecuteContext extends BaseExecuteContext implements IExecuteFunctions { readonly helpers: IExecuteFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts b/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts index acf342e0ac..d9d0561824 100644 --- a/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/execute-single-context.ts @@ -13,9 +13,6 @@ import type { } from 'n8n-workflow'; import { ApplicationError, createDeferredPromise, NodeConnectionType } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { getRequestHelperFunctions, returnJsonArray } from '@/node-execute-functions'; - import { BaseExecuteContext } from './base-execute-context'; import { assertBinaryData, @@ -23,6 +20,8 @@ import { getBinaryDataBuffer, getBinaryHelperFunctions, } from './utils/binary-helper-functions'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; export class ExecuteSingleContext extends BaseExecuteContext implements IExecuteSingleFunctions { readonly helpers: IExecuteSingleFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/hook-context.ts b/packages/core/src/execution-engine/node-execution-context/hook-context.ts index 28dbac802c..0fdf1c5954 100644 --- a/packages/core/src/execution-engine/node-execution-context/hook-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/hook-context.ts @@ -11,14 +11,9 @@ import type { } from 'n8n-workflow'; import { ApplicationError } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - getNodeWebhookUrl, - getRequestHelperFunctions, - getWebhookDescription, -} from '@/node-execute-functions'; - import { NodeExecutionContext } from './node-execution-context'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { getNodeWebhookUrl, getWebhookDescription } from './utils/webhook-helper-functions'; export class HookContext extends NodeExecutionContext implements IHookFunctions { readonly helpers: IHookFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/index.ts b/packages/core/src/execution-engine/node-execution-context/index.ts index 0c6843d525..dcac1576d1 100644 --- a/packages/core/src/execution-engine/node-execution-context/index.ts +++ b/packages/core/src/execution-engine/node-execution-context/index.ts @@ -1,18 +1,18 @@ -// eslint-disable-next-line import/no-cycle export { CredentialTestContext } from './credentials-test-context'; -// eslint-disable-next-line import/no-cycle export { ExecuteContext } from './execute-context'; export { ExecuteSingleContext } from './execute-single-context'; export { HookContext } from './hook-context'; export { LoadOptionsContext } from './load-options-context'; export { LocalLoadOptionsContext } from './local-load-options-context'; export { PollContext } from './poll-context'; -// eslint-disable-next-line import/no-cycle export { SupplyDataContext } from './supply-data-context'; export { TriggerContext } from './trigger-context'; export { WebhookContext } from './webhook-context'; +export { constructExecutionMetaData } from './utils/construct-execution-metadata'; export { getAdditionalKeys } from './utils/get-additional-keys'; +export { normalizeItems } from './utils/normalize-items'; export { parseIncomingMessage } from './utils/parse-incoming-message'; -export { parseRequestObject } from './utils/parse-request-object'; +export { parseRequestObject } from './utils/request-helper-functions'; +export { returnJsonArray } from './utils/return-json-array'; export * from './utils/binary-helper-functions'; diff --git a/packages/core/src/execution-engine/node-execution-context/load-options-context.ts b/packages/core/src/execution-engine/node-execution-context/load-options-context.ts index 30c31eacce..e39566edd3 100644 --- a/packages/core/src/execution-engine/node-execution-context/load-options-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/load-options-context.ts @@ -9,11 +9,10 @@ import type { Workflow, } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { getRequestHelperFunctions, getSSHTunnelFunctions } from '@/node-execute-functions'; - import { NodeExecutionContext } from './node-execution-context'; import { extractValue } from './utils/extract-value'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { getSSHTunnelFunctions } from './utils/ssh-tunnel-helper-functions'; export class LoadOptionsContext extends NodeExecutionContext implements ILoadOptionsFunctions { readonly helpers: ILoadOptionsFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/poll-context.ts b/packages/core/src/execution-engine/node-execution-context/poll-context.ts index 793cd8c671..b96991a5d5 100644 --- a/packages/core/src/execution-engine/node-execution-context/poll-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/poll-context.ts @@ -9,15 +9,11 @@ import type { } from 'n8n-workflow'; import { ApplicationError, createDeferredPromise } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - getRequestHelperFunctions, - getSchedulingFunctions, - returnJsonArray, -} from '@/node-execute-functions'; - import { NodeExecutionContext } from './node-execution-context'; import { getBinaryHelperFunctions } from './utils/binary-helper-functions'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; +import { getSchedulingFunctions } from './utils/scheduling-helper-functions'; const throwOnEmit = () => { throw new ApplicationError('Overwrite PollContext.__emit function'); diff --git a/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts b/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts index 33e2723a20..40d098bedd 100644 --- a/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/supply-data-context.ts @@ -19,16 +19,6 @@ import type { } from 'n8n-workflow'; import { createDeferredPromise } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - constructExecutionMetaData, - copyInputItems, - getRequestHelperFunctions, - getSSHTunnelFunctions, - normalizeItems, - returnJsonArray, -} from '@/node-execute-functions'; - import { BaseExecuteContext } from './base-execute-context'; import { assertBinaryData, @@ -36,9 +26,16 @@ import { getBinaryDataBuffer, getBinaryHelperFunctions, } from './utils/binary-helper-functions'; +import { constructExecutionMetaData } from './utils/construct-execution-metadata'; +import { copyInputItems } from './utils/copy-input-items'; import { getDeduplicationHelperFunctions } from './utils/deduplication-helper-functions'; import { getFileSystemHelperFunctions } from './utils/file-system-helper-functions'; +// eslint-disable-next-line import/no-cycle import { getInputConnectionData } from './utils/get-input-connection-data'; +import { normalizeItems } from './utils/normalize-items'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; +import { getSSHTunnelFunctions } from './utils/ssh-tunnel-helper-functions'; export class SupplyDataContext extends BaseExecuteContext implements ISupplyDataFunctions { readonly helpers: ISupplyDataFunctions['helpers']; diff --git a/packages/core/src/execution-engine/node-execution-context/trigger-context.ts b/packages/core/src/execution-engine/node-execution-context/trigger-context.ts index 01bdaf6b03..2ecf25479b 100644 --- a/packages/core/src/execution-engine/node-execution-context/trigger-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/trigger-context.ts @@ -9,16 +9,12 @@ import type { } from 'n8n-workflow'; import { ApplicationError, createDeferredPromise } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - getRequestHelperFunctions, - getSchedulingFunctions, - getSSHTunnelFunctions, - returnJsonArray, -} from '@/node-execute-functions'; - import { NodeExecutionContext } from './node-execution-context'; import { getBinaryHelperFunctions } from './utils/binary-helper-functions'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; +import { getSchedulingFunctions } from './utils/scheduling-helper-functions'; +import { getSSHTunnelFunctions } from './utils/ssh-tunnel-helper-functions'; const throwOnEmit = () => { throw new ApplicationError('Overwrite TriggerContext.emit function'); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/construct-execution-metadata.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/construct-execution-metadata.test.ts new file mode 100644 index 0000000000..01df682602 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/construct-execution-metadata.test.ts @@ -0,0 +1,44 @@ +import type { INodeExecutionData, IPairedItemData, NodeExecutionWithMetadata } from 'n8n-workflow'; + +import { constructExecutionMetaData } from '../construct-execution-metadata'; + +describe('constructExecutionMetaData', () => { + const tests: Array<{ + description: string; + inputData: INodeExecutionData[]; + itemData: IPairedItemData | IPairedItemData[]; + expected: NodeExecutionWithMetadata[]; + }> = [ + { + description: 'should add pairedItem to single input data', + inputData: [{ json: { name: 'John' } }], + itemData: { item: 0 }, + expected: [{ json: { name: 'John' }, pairedItem: { item: 0 } }], + }, + { + description: 'should add pairedItem to multiple input data with different properties', + inputData: [{ json: { name: 'John' } }, { json: { name: 'Jane' } }], + itemData: [{ item: 0 }, { item: 1 }], + expected: [ + { json: { name: 'John' }, pairedItem: [{ item: 0 }, { item: 1 }] }, + { json: { name: 'Jane' }, pairedItem: [{ item: 0 }, { item: 1 }] }, + ], + }, + { + description: 'should handle empty input data and itemData', + inputData: [], + itemData: [], + expected: [], + }, + { + description: 'should handle multiple pairedItem with single input data', + inputData: [{ json: { name: 'John' } }], + itemData: [{ item: 0 }, { item: 1 }], + expected: [{ json: { name: 'John' }, pairedItem: [{ item: 0 }, { item: 1 }] }], + }, + ]; + test.each(tests)('$description', ({ inputData, itemData, expected }) => { + const result = constructExecutionMetaData(inputData, { itemData }); + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/copy-input-items.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/copy-input-items.test.ts new file mode 100644 index 0000000000..7f74be9dc4 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/copy-input-items.test.ts @@ -0,0 +1,49 @@ +import { copyInputItems } from '../copy-input-items'; + +describe('copyInputItems', () => { + it('should pick only selected properties', () => { + const output = copyInputItems( + [ + { + json: { + a: 1, + b: true, + c: {}, + }, + }, + ], + ['a'], + ); + expect(output).toEqual([{ a: 1 }]); + }); + + it('should convert undefined to null', () => { + const output = copyInputItems( + [ + { + json: { + a: undefined, + }, + }, + ], + ['a'], + ); + expect(output).toEqual([{ a: null }]); + }); + + it('should clone objects', () => { + const input = { + a: { b: 5 }, + }; + const output = copyInputItems( + [ + { + json: input, + }, + ], + ['a'], + ); + expect(output[0].a).toEqual(input.a); + expect(output[0].a === input.a).toEqual(false); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/normalize-items.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/normalize-items.test.ts new file mode 100644 index 0000000000..2664e1bf08 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/normalize-items.test.ts @@ -0,0 +1,110 @@ +import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; +import { ApplicationError } from 'n8n-workflow'; + +import { normalizeItems } from '../normalize-items'; + +describe('normalizeItems', () => { + describe('should handle', () => { + const successTests: Array<{ + description: string; + input: INodeExecutionData | INodeExecutionData[]; + expected: INodeExecutionData[]; + }> = [ + { + description: 'single object without json key', + input: { key: 'value' } as unknown as INodeExecutionData, + expected: [{ json: { key: 'value' } }], + }, + { + description: 'array of objects without json key', + input: [{ key1: 'value1' }, { key2: 'value2' }] as unknown as INodeExecutionData[], + expected: [{ json: { key1: 'value1' } }, { json: { key2: 'value2' } }], + }, + { + description: 'single object with json key', + input: { json: { key: 'value' } } as INodeExecutionData, + expected: [{ json: { key: 'value' } }], + }, + { + description: 'array of objects with json key', + input: [{ json: { key1: 'value1' } }, { json: { key2: 'value2' } }] as INodeExecutionData[], + expected: [{ json: { key1: 'value1' } }, { json: { key2: 'value2' } }], + }, + { + description: 'array of objects with binary data', + input: [ + { json: {}, binary: { data: { data: 'binary1', mimeType: 'mime1' } } }, + { json: {}, binary: { data: { data: 'binary2', mimeType: 'mime2' } } }, + ], + expected: [ + { json: {}, binary: { data: { data: 'binary1', mimeType: 'mime1' } } }, + { json: {}, binary: { data: { data: 'binary2', mimeType: 'mime2' } } }, + ], + }, + { + description: 'object with null or undefined values', + input: { key: null, another: undefined } as unknown as INodeExecutionData, + expected: [{ json: { key: null, another: undefined } }], + }, + { + description: 'array with mixed non-standard objects', + input: [{ custom: 'value1' }, { another: 'value2' }] as unknown as INodeExecutionData[], + expected: [{ json: { custom: 'value1' } }, { json: { another: 'value2' } }], + }, + { + description: 'empty object', + input: {} as INodeExecutionData, + expected: [{ json: {} }], + }, + { + description: 'array with primitive values', + input: [1, 'string', true] as unknown as INodeExecutionData[], + expected: [ + { json: 1 }, + { json: 'string' }, + { json: true }, + ] as unknown as INodeExecutionData[], + }, + ]; + test.each(successTests)('$description', ({ input, expected }) => { + const result = normalizeItems(input); + expect(result).toEqual(expected); + }); + }); + + describe('should throw error', () => { + const errorTests: Array<{ + description: string; + input: INodeExecutionData[]; + }> = [ + { + description: 'for inconsistent items with some having json key', + input: [{ json: { key1: 'value1' } }, { key2: 'value2' } as unknown as INodeExecutionData], + }, + { + description: 'for inconsistent items with some having binary key', + input: [ + { json: {}, binary: { data: { data: 'binary1', mimeType: 'mime1' } } }, + { key: 'value' } as unknown as INodeExecutionData, + ], + }, + { + description: 'when mixing json and non-json objects with non-json properties', + input: [ + { json: { key1: 'value1' } }, + { other: 'value', custom: 'prop' } as unknown as INodeExecutionData, + ], + }, + { + description: 'when mixing binary and non-binary objects', + input: [ + { json: {}, binary: { data: { data: 'binarydata' } as IBinaryData } }, + { custom: 'value' } as unknown as INodeExecutionData, + ], + }, + ]; + test.each(errorTests)('$description', ({ input }) => { + expect(() => normalizeItems(input)).toThrow(new ApplicationError('Inconsistent item format')); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts deleted file mode 100644 index 9812057a4b..0000000000 --- a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/parse-request-object.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import FormData from 'form-data'; -import type { Agent } from 'https'; -import { mock } from 'jest-mock-extended'; -import type { IHttpRequestMethods, IRequestOptions } from 'n8n-workflow'; -import type { SecureContextOptions } from 'tls'; - -import { parseRequestObject } from '../parse-request-object'; - -describe('parseRequestObject', () => { - test('should handle basic request options', async () => { - const axiosOptions = await parseRequestObject({ - url: 'https://example.com', - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: { key: 'value' }, - }); - - expect(axiosOptions).toEqual( - expect.objectContaining({ - url: 'https://example.com', - method: 'POST', - headers: { accept: '*/*', 'content-type': 'application/json' }, - data: { key: 'value' }, - maxRedirects: 0, - }), - ); - }); - - test('should set correct headers for FormData', async () => { - const formData = new FormData(); - formData.append('key', 'value'); - - const axiosOptions = await parseRequestObject({ - url: 'https://example.com', - formData, - headers: { - 'content-type': 'multipart/form-data', - }, - }); - - expect(axiosOptions.headers).toMatchObject({ - accept: '*/*', - 'content-length': 163, - 'content-type': expect.stringMatching(/^multipart\/form-data; boundary=/), - }); - - expect(axiosOptions.data).toBeInstanceOf(FormData); - }); - - test('should not use Host header for SNI', async () => { - const axiosOptions = await parseRequestObject({ - url: 'https://example.de/foo/bar', - headers: { Host: 'other.host.com' }, - }); - expect((axiosOptions.httpsAgent as Agent).options.servername).toEqual('example.de'); - }); - - describe('should set SSL certificates', () => { - const agentOptions: SecureContextOptions = { - ca: '-----BEGIN CERTIFICATE-----\nTEST\n-----END CERTIFICATE-----', - }; - const requestObject: IRequestOptions = { - method: 'GET', - uri: 'https://example.de', - agentOptions, - }; - - test('on regular requests', async () => { - const axiosOptions = await parseRequestObject(requestObject); - expect((axiosOptions.httpsAgent as Agent).options).toEqual({ - servername: 'example.de', - ...agentOptions, - noDelay: true, - path: null, - }); - }); - - test('on redirected requests', async () => { - const axiosOptions = await parseRequestObject(requestObject); - expect(axiosOptions.beforeRedirect).toBeDefined; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const redirectOptions: Record = { agents: {}, hostname: 'example.de' }; - axiosOptions.beforeRedirect!(redirectOptions, mock()); - expect(redirectOptions.agent).toEqual(redirectOptions.agents.https); - expect((redirectOptions.agent as Agent).options).toEqual({ - servername: 'example.de', - ...agentOptions, - noDelay: true, - path: null, - }); - }); - }); - - describe('when followRedirect is true', () => { - test.each(['GET', 'HEAD'] as IHttpRequestMethods[])( - 'should set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followRedirect: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(1234); - }, - ); - - test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'should not set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followRedirect: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(0); - }, - ); - }); - - describe('when followAllRedirects is true', () => { - test.each(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( - 'should set maxRedirects on %s ', - async (method) => { - const axiosOptions = await parseRequestObject({ - method, - followAllRedirects: true, - maxRedirects: 1234, - }); - expect(axiosOptions.maxRedirects).toEqual(1234); - }, - ); - }); -}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts new file mode 100644 index 0000000000..a8dcaf08aa --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts @@ -0,0 +1,865 @@ +import FormData from 'form-data'; +import type { Agent } from 'https'; +import { mock } from 'jest-mock-extended'; +import type { + IHttpRequestMethods, + IHttpRequestOptions, + INode, + IRequestOptions, + IWorkflowExecuteAdditionalData, + PaginationOptions, + Workflow, +} from 'n8n-workflow'; +import nock from 'nock'; +import type { SecureContextOptions } from 'tls'; + +import type { ExecutionLifecycleHooks } from '@/execution-engine/execution-lifecycle-hooks'; + +import { + applyPaginationRequestData, + convertN8nRequestToAxios, + createFormDataObject, + httpRequest, + invokeAxios, + parseRequestObject, + proxyRequestToAxios, + removeEmptyBody, +} from '../request-helper-functions'; + +describe('Request Helper Functions', () => { + describe('proxyRequestToAxios', () => { + const baseUrl = 'http://example.de'; + const workflow = mock(); + const hooks = mock(); + const additionalData = mock({ hooks }); + const node = mock(); + + beforeEach(() => { + hooks.runHook.mockClear(); + }); + + test('should rethrow an error with `status` property', async () => { + nock(baseUrl).get('/test').reply(400); + + try { + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + } catch (error) { + expect(error.status).toEqual(400); + } + }); + + test('should not throw if the response status is 200', async () => { + nock(baseUrl).get('/test').reply(200); + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + test('should throw if the response status is 403', async () => { + const headers = { 'content-type': 'text/plain' }; + nock(baseUrl).get('/test').reply(403, 'Forbidden', headers); + try { + await proxyRequestToAxios(workflow, additionalData, node, `${baseUrl}/test`); + } catch (error) { + expect(error.statusCode).toEqual(403); + expect(error.request).toBeUndefined(); + expect(error.response).toMatchObject({ headers, status: 403 }); + expect(error.options).toMatchObject({ + headers: { Accept: '*/*' }, + method: 'get', + url: 'http://example.de/test', + }); + expect(error.config).toBeUndefined(); + expect(error.message).toEqual('403 - "Forbidden"'); + } + expect(hooks.runHook).not.toHaveBeenCalled(); + }); + + test('should not throw if the response status is 404, but `simple` option is set to `false`', async () => { + nock(baseUrl).get('/test').reply(404, 'Not Found'); + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/test`, + simple: false, + }); + + expect(response).toEqual('Not Found'); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + test('should return full response when `resolveWithFullResponse` is set to true', async () => { + nock(baseUrl).get('/test').reply(404, 'Not Found'); + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/test`, + resolveWithFullResponse: true, + simple: false, + }); + + expect(response).toMatchObject({ + body: 'Not Found', + headers: {}, + statusCode: 404, + statusMessage: 'Not Found', + }); + expect(hooks.runHook).toHaveBeenCalledWith('nodeFetchedData', [workflow.id, node]); + }); + + describe('redirects', () => { + test('should forward authorization header', async () => { + nock(baseUrl).get('/redirect').reply(301, '', { Location: 'https://otherdomain.com/test' }); + nock('https://otherdomain.com') + .get('/test') + .reply(200, function () { + return this.req.headers; + }); + + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + auth: { + username: 'testuser', + password: 'testpassword', + }, + headers: { + 'X-Other-Header': 'otherHeaderContent', + }, + resolveWithFullResponse: true, + }); + + expect(response.statusCode).toBe(200); + const forwardedHeaders = JSON.parse(response.body); + expect(forwardedHeaders.authorization).toBe('Basic dGVzdHVzZXI6dGVzdHBhc3N3b3Jk'); + expect(forwardedHeaders['x-other-header']).toBe('otherHeaderContent'); + }); + + test('should follow redirects by default', async () => { + nock(baseUrl) + .get('/redirect') + .reply(301, '', { Location: `${baseUrl}/test` }); + nock(baseUrl).get('/test').reply(200, 'Redirected'); + + const response = await proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + resolveWithFullResponse: true, + }); + + expect(response).toMatchObject({ + body: 'Redirected', + headers: {}, + statusCode: 200, + }); + }); + + test('should not follow redirects when configured', async () => { + nock(baseUrl) + .get('/redirect') + .reply(301, '', { Location: `${baseUrl}/test` }); + nock(baseUrl).get('/test').reply(200, 'Redirected'); + + await expect( + proxyRequestToAxios(workflow, additionalData, node, { + url: `${baseUrl}/redirect`, + resolveWithFullResponse: true, + followRedirect: false, + }), + ).rejects.toThrowError(expect.objectContaining({ statusCode: 301 })); + }); + }); + }); + + describe('invokeAxios', () => { + const baseUrl = 'http://example.de'; + + beforeEach(() => { + nock.cleanAll(); + jest.clearAllMocks(); + }); + + it('should throw error for non-401 status codes', async () => { + nock(baseUrl).get('/test').reply(500, {}); + + await expect(invokeAxios({ url: `${baseUrl}/test` })).rejects.toThrow( + 'Request failed with status code 500', + ); + }); + + it('should throw error on 401 without digest auth challenge', async () => { + nock(baseUrl).get('/test').reply(401, {}); + + await expect( + invokeAxios( + { + url: `${baseUrl}/test`, + }, + { sendImmediately: false }, + ), + ).rejects.toThrow('Request failed with status code 401'); + }); + + it('should make successful requests', async () => { + nock(baseUrl).get('/test').reply(200, { success: true }); + + const response = await invokeAxios({ + url: `${baseUrl}/test`, + }); + + expect(response.status).toBe(200); + expect(response.data).toEqual({ success: true }); + }); + + it('should handle digest auth when receiving 401 with nonce', async () => { + nock(baseUrl) + .get('/test') + .matchHeader('authorization', 'Basic dXNlcjpwYXNz') + .once() + .reply(401, {}, { 'www-authenticate': 'Digest realm="test", nonce="abc123", qop="auth"' }); + + nock(baseUrl) + .get('/test') + .matchHeader( + 'authorization', + /^Digest username="user",realm="test",nonce="abc123",uri="\/test",qop="auth",algorithm="MD5",response="[0-9a-f]{32}"/, + ) + .reply(200, { success: true }); + + const response = await invokeAxios( + { + url: `${baseUrl}/test`, + auth: { + username: 'user', + password: 'pass', + }, + }, + { sendImmediately: false }, + ); + + expect(response.status).toBe(200); + expect(response.data).toEqual({ success: true }); + }); + }); + + describe('removeEmptyBody', () => { + test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( + 'Should remove empty body for %s', + async (method) => { + const requestOptions = { + method, + body: {}, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual(undefined); + }, + ); + + test.each(['GET', 'HEAD', 'OPTIONS'] as IHttpRequestMethods[])( + 'Should not remove non-empty body for %s', + async (method) => { + const requestOptions = { + method, + body: { test: true }, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual({ test: true }); + }, + ); + + test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'Should not remove empty body for %s', + async (method) => { + const requestOptions = { + method, + body: {}, + } as IHttpRequestOptions | IRequestOptions; + removeEmptyBody(requestOptions); + expect(requestOptions.body).toEqual({}); + }, + ); + }); + + describe('parseRequestObject', () => { + test('should handle basic request options', async () => { + const axiosOptions = await parseRequestObject({ + url: 'https://example.com', + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: { key: 'value' }, + }); + + expect(axiosOptions).toEqual( + expect.objectContaining({ + url: 'https://example.com', + method: 'POST', + headers: { accept: '*/*', 'content-type': 'application/json' }, + data: { key: 'value' }, + maxRedirects: 0, + }), + ); + }); + + test('should set correct headers for FormData', async () => { + const formData = new FormData(); + formData.append('key', 'value'); + + const axiosOptions = await parseRequestObject({ + url: 'https://example.com', + formData, + headers: { + 'content-type': 'multipart/form-data', + }, + }); + + expect(axiosOptions.headers).toMatchObject({ + accept: '*/*', + 'content-length': 163, + 'content-type': expect.stringMatching(/^multipart\/form-data; boundary=/), + }); + + expect(axiosOptions.data).toBeInstanceOf(FormData); + }); + + test('should not use Host header for SNI', async () => { + const axiosOptions = await parseRequestObject({ + url: 'https://example.de/foo/bar', + headers: { Host: 'other.host.com' }, + }); + expect((axiosOptions.httpsAgent as Agent).options.servername).toEqual('example.de'); + }); + + describe('should set SSL certificates', () => { + const agentOptions: SecureContextOptions = { + ca: '-----BEGIN CERTIFICATE-----\nTEST\n-----END CERTIFICATE-----', + }; + const requestObject: IRequestOptions = { + method: 'GET', + uri: 'https://example.de', + agentOptions, + }; + + test('on regular requests', async () => { + const axiosOptions = await parseRequestObject(requestObject); + expect((axiosOptions.httpsAgent as Agent).options).toEqual({ + servername: 'example.de', + ...agentOptions, + noDelay: true, + path: null, + }); + }); + + test('on redirected requests', async () => { + const axiosOptions = await parseRequestObject(requestObject); + expect(axiosOptions.beforeRedirect).toBeDefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const redirectOptions: Record = { agents: {}, hostname: 'example.de' }; + axiosOptions.beforeRedirect!(redirectOptions, mock()); + expect(redirectOptions.agent).toEqual(redirectOptions.agents.https); + expect((redirectOptions.agent as Agent).options).toEqual({ + servername: 'example.de', + ...agentOptions, + noDelay: true, + path: null, + }); + }); + }); + + describe('when followRedirect is true', () => { + test.each(['GET', 'HEAD'] as IHttpRequestMethods[])( + 'should set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followRedirect: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(1234); + }, + ); + + test.each(['POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'should not set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followRedirect: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(0); + }, + ); + }); + + describe('when followAllRedirects is true', () => { + test.each(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] as IHttpRequestMethods[])( + 'should set maxRedirects on %s ', + async (method) => { + const axiosOptions = await parseRequestObject({ + method, + followAllRedirects: true, + maxRedirects: 1234, + }); + expect(axiosOptions.maxRedirects).toEqual(1234); + }, + ); + }); + }); + + describe('createFormDataObject', () => { + test('should create FormData with simple key-value pairs', () => { + const data = { key1: 'value1', key2: 'value2' }; + const formData = createFormDataObject(data); + + expect(formData).toBeInstanceOf(FormData); + + const formDataEntries: string[] = []; + formData.getHeaders(); // Ensures form data is processed + + formData.on('data', (chunk) => { + formDataEntries.push(chunk.toString()); + }); + }); + + test('should handle array values', () => { + const data = { files: ['file1.txt', 'file2.txt'] }; + const formData = createFormDataObject(data); + + expect(formData).toBeInstanceOf(FormData); + }); + + test('should handle complex form data with options', () => { + const data = { + file: { + value: Buffer.from('test content'), + options: { + filename: 'test.txt', + contentType: 'text/plain', + }, + }, + }; + + const formData = createFormDataObject(data); + + expect(formData).toBeInstanceOf(FormData); + }); + }); + + describe('convertN8nRequestToAxios', () => { + test('should convert basic HTTP request options', () => { + const requestOptions: IHttpRequestOptions = { + method: 'GET', + url: 'https://example.com', + headers: { 'Custom-Header': 'test' }, + qs: { param1: 'value1' }, + }; + + const axiosConfig = convertN8nRequestToAxios(requestOptions); + + expect(axiosConfig).toEqual( + expect.objectContaining({ + method: 'GET', + url: 'https://example.com', + headers: expect.objectContaining({ + 'Custom-Header': 'test', + 'User-Agent': 'n8n', + }), + params: { param1: 'value1' }, + }), + ); + }); + + test('should handle body and content type', () => { + const requestOptions: IHttpRequestOptions = { + method: 'POST', + url: 'https://example.com', + body: { key: 'value' }, + headers: { 'content-type': 'application/json' }, + }; + + const axiosConfig = convertN8nRequestToAxios(requestOptions); + + expect(axiosConfig).toEqual( + expect.objectContaining({ + method: 'POST', + data: { key: 'value' }, + headers: expect.objectContaining({ + 'content-type': 'application/json', + }), + }), + ); + }); + + test('should handle form data', () => { + const formData = new FormData(); + formData.append('key', 'value'); + + const requestOptions: IHttpRequestOptions = { + method: 'POST', + url: 'https://example.com', + body: formData, + }; + + const axiosConfig = convertN8nRequestToAxios(requestOptions); + + expect(axiosConfig).toEqual( + expect.objectContaining({ + method: 'POST', + data: formData, + headers: expect.objectContaining({ + ...formData.getHeaders(), + 'User-Agent': 'n8n', + }), + }), + ); + }); + + test('should handle disable follow redirect', () => { + const requestOptions: IHttpRequestOptions = { + method: 'GET', + url: 'https://example.com', + disableFollowRedirect: true, + }; + + const axiosConfig = convertN8nRequestToAxios(requestOptions); + + expect(axiosConfig.maxRedirects).toBe(0); + }); + + test('should handle SSL certificate validation', () => { + const requestOptions: IHttpRequestOptions = { + method: 'GET', + url: 'https://example.com', + skipSslCertificateValidation: true, + }; + + const axiosConfig = convertN8nRequestToAxios(requestOptions); + + expect(axiosConfig.httpsAgent?.options.rejectUnauthorized).toBe(false); + }); + }); + + describe('applyPaginationRequestData', () => { + test('should merge pagination request data with original request options', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + qs: { page: 1 }, + headers: { 'X-Original-Header': 'original' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + headers: { 'X-Pagination-Header': 'pagination' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'GET', + qs: { page: 1 }, + headers: { + 'X-Original-Header': 'original', + 'X-Pagination-Header': 'pagination', + }, + body: { key: 'value' }, + }); + }); + + test('should handle formData correctly', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + formData: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + formData: { key: 'value', original: 'data' }, + }); + }); + + test('should handle form data correctly', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + form: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + form: { key: 'value', original: 'data' }, + }); + }); + + test('should prefer pagination body over original body', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + body: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + body: { key: 'value', original: 'data' }, + }); + }); + + test('should merge complex request options', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + qs: { page: 1, limit: 10 }, + headers: { 'X-Original-Header': 'original' }, + body: { filter: 'active' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + headers: { 'X-Pagination-Header': 'pagination' }, + qs: { offset: 20 }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'GET', + qs: { offset: 20, limit: 10, page: 1 }, + headers: { + 'X-Original-Header': 'original', + 'X-Pagination-Header': 'pagination', + }, + body: { key: 'value', filter: 'active' }, + }); + }); + + test('should handle edge cases with empty pagination data', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + }; + + const paginationRequestData: PaginationOptions['request'] = {}; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://original.com/api', + method: 'GET', + }); + }); + }); + + describe('httpRequest', () => { + const baseUrl = 'https://example.com'; + + beforeEach(() => { + nock.cleanAll(); + }); + + test('should make a simple GET request', async () => { + const scope = nock(baseUrl) + .get('/users') + .reply(200, { users: ['John', 'Jane'] }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/users`, + }); + + expect(response).toEqual({ users: ['John', 'Jane'] }); + scope.done(); + }); + + test('should make a POST request with JSON body', async () => { + const requestBody = { name: 'John', age: 30 }; + const scope = nock(baseUrl) + .post('/users', requestBody) + .reply(201, { id: '123', ...requestBody }); + + const response = await httpRequest({ + method: 'POST', + url: `${baseUrl}/users`, + body: requestBody, + json: true, + }); + + expect(response).toEqual({ id: '123', name: 'John', age: 30 }); + scope.done(); + }); + + test('should return full response when returnFullResponse is true', async () => { + const scope = nock(baseUrl).get('/data').reply( + 200, + { key: 'value' }, + { + 'X-Custom-Header': 'test-header', + }, + ); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/data`, + returnFullResponse: true, + }); + + expect(response).toEqual({ + body: { key: 'value' }, + headers: expect.objectContaining({ + 'x-custom-header': 'test-header', + }), + statusCode: 200, + statusMessage: 'OK', + }); + scope.done(); + }); + + test('should handle form data request', async () => { + const formData = new FormData(); + formData.append('file', 'test content', 'file.txt'); + + const scope = nock(baseUrl) + .post('/upload') + .matchHeader('content-type', /multipart\/form-data; boundary=/) + .reply(200, { success: true }); + + const response = await httpRequest({ + method: 'POST', + url: `${baseUrl}/upload`, + body: formData, + }); + + expect(response).toEqual({ success: true }); + scope.done(); + }); + + test('should handle query parameters', async () => { + const scope = nock(baseUrl) + .get('/search') + .query({ q: 'test', page: '1' }) + .reply(200, { results: ['result1', 'result2'] }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/search`, + qs: { q: 'test', page: '1' }, + }); + + expect(response).toEqual({ results: ['result1', 'result2'] }); + scope.done(); + }); + + test('should ignore HTTP status errors when configured', async () => { + const scope = nock(baseUrl).get('/error').reply(500, { error: 'Internal Server Error' }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/error`, + ignoreHttpStatusErrors: true, + }); + + expect(response).toEqual({ error: 'Internal Server Error' }); + scope.done(); + }); + + test('should handle different array formats in query parameters', async () => { + const scope = nock(baseUrl) + .get('/list') + .query({ + tags: ['tag1', 'tag2'], + categories: ['cat1', 'cat2'], + }) + .reply(200, { success: true }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/list`, + qs: { + tags: ['tag1', 'tag2'], + categories: ['cat1', 'cat2'], + }, + arrayFormat: 'indices', + }); + + expect(response).toEqual({ success: true }); + scope.done(); + }); + + test('should remove empty body for GET requests', async () => { + const scope = nock(baseUrl).get('/data').reply(200, { success: true }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/data`, + body: {}, + }); + + expect(response).toEqual({ success: true }); + scope.done(); + }); + + test('should set default user agent', async () => { + const scope = nock(baseUrl, { + reqheaders: { + 'user-agent': 'n8n', + }, + }) + .get('/test') + .reply(200, { success: true }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/test`, + }); + + expect(response).toEqual({ success: true }); + scope.done(); + }); + + test('should respect custom headers', async () => { + const scope = nock(baseUrl, { + reqheaders: { + 'X-Custom-Header': 'custom-value', + 'user-agent': 'n8n', + }, + }) + .get('/test') + .reply(200, { success: true }); + + const response = await httpRequest({ + method: 'GET', + url: `${baseUrl}/test`, + headers: { 'X-Custom-Header': 'custom-value' }, + }); + + expect(response).toEqual({ success: true }); + scope.done(); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/return-json-array.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/return-json-array.test.ts new file mode 100644 index 0000000000..13c087047c --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/return-json-array.test.ts @@ -0,0 +1,34 @@ +import { returnJsonArray } from '../return-json-array'; + +describe('returnJsonArray', () => { + test.each([ + { + input: { name: 'John', age: 30 }, + expected: [{ json: { name: 'John', age: 30 } }], + description: 'should convert a single object to an array with json key', + }, + { + input: [{ name: 'John' }, { name: 'Jane' }], + expected: [{ json: { name: 'John' } }, { json: { name: 'Jane' } }], + description: 'should return an array of objects with json key', + }, + { + input: [{ json: { name: 'John' }, additionalProp: 'value' }], + expected: [{ json: { name: 'John' }, additionalProp: 'value' }], + description: 'should preserve existing json key in object', + }, + { + input: [], + expected: [], + description: 'should handle empty array input', + }, + { + input: [{ name: 'John' }, { json: { name: 'Jane' }, additionalProp: 'value' }], + expected: [{ json: { name: 'John' } }, { json: { name: 'Jane' }, additionalProp: 'value' }], + description: 'should handle mixed input with some objects having json key', + }, + ])('$description', ({ input, expected }) => { + const result = returnJsonArray(input); + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/scheduling-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/scheduling-helper-functions.test.ts new file mode 100644 index 0000000000..e9a5e63f58 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/scheduling-helper-functions.test.ts @@ -0,0 +1,31 @@ +import { mock } from 'jest-mock-extended'; +import type { Workflow } from 'n8n-workflow'; + +import { mockInstance } from '@test/utils'; + +import { ScheduledTaskManager } from '../../../scheduled-task-manager'; +import { getSchedulingFunctions } from '../scheduling-helper-functions'; + +describe('getSchedulingFunctions', () => { + const workflow = mock({ id: 'test-workflow' }); + const cronExpression = '* * * * * 0'; + const onTick = jest.fn(); + const scheduledTaskManager = mockInstance(ScheduledTaskManager); + const schedulingFunctions = getSchedulingFunctions(workflow); + + it('should return scheduling functions', () => { + expect(typeof schedulingFunctions.registerCron).toBe('function'); + }); + + describe('registerCron', () => { + it('should invoke scheduledTaskManager.registerCron', () => { + schedulingFunctions.registerCron(cronExpression, onTick); + + expect(scheduledTaskManager.registerCron).toHaveBeenCalledWith( + workflow, + cronExpression, + onTick, + ); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/ssh-tunnel-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/ssh-tunnel-helper-functions.test.ts new file mode 100644 index 0000000000..5300a7d3c2 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/ssh-tunnel-helper-functions.test.ts @@ -0,0 +1,25 @@ +import { mock } from 'jest-mock-extended'; +import type { SSHCredentials } from 'n8n-workflow'; + +import { mockInstance } from '@test/utils'; + +import { SSHClientsManager } from '../../../ssh-clients-manager'; +import { getSSHTunnelFunctions } from '../ssh-tunnel-helper-functions'; + +describe('getSSHTunnelFunctions', () => { + const credentials = mock(); + const sshClientsManager = mockInstance(SSHClientsManager); + const sshTunnelFunctions = getSSHTunnelFunctions(); + + it('should return SSH tunnel functions', () => { + expect(typeof sshTunnelFunctions.getSSHClient).toBe('function'); + }); + + describe('getSSHClient', () => { + it('should invoke sshClientsManager.getClient', async () => { + await sshTunnelFunctions.getSSHClient(credentials); + + expect(sshClientsManager.getClient).toHaveBeenCalledWith(credentials); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/webhook-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/webhook-helper-functions.test.ts new file mode 100644 index 0000000000..f8b20297f3 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/webhook-helper-functions.test.ts @@ -0,0 +1,150 @@ +import { mock } from 'jest-mock-extended'; +import type { + WebhookType, + Workflow, + INode, + IWebhookDescription, + INodeType, + INodeTypes, + Expression, + IWorkflowExecuteAdditionalData, +} from 'n8n-workflow'; + +import { getWebhookDescription, getNodeWebhookUrl } from '../webhook-helper-functions'; + +describe('Webhook Helper Functions', () => { + const nodeTypes = mock(); + const expression = mock(); + const workflow = mock({ id: 'workflow-id', expression, nodeTypes }); + const nodeType = mock(); + const node = mock({ name: 'test-node' }); + + beforeEach(() => jest.resetAllMocks()); + + describe('getWebhookDescription', () => { + const tests: Array<{ + description: string; + name: WebhookType; + webhooks: IWebhookDescription[] | undefined; + expected: IWebhookDescription | undefined; + }> = [ + { + description: 'should return undefined for invalid webhook type', + name: 'invalid' as WebhookType, + webhooks: [ + { + name: 'default', + httpMethod: 'POST', + path: 'webhook', + }, + ], + expected: undefined, + }, + { + description: 'should return undefined when node has no webhooks', + name: 'default', + webhooks: undefined, + expected: undefined, + }, + { + description: 'should return webhook description when webhook exists', + name: 'default', + webhooks: [ + { + name: 'default', + httpMethod: 'POST', + path: 'webhook', + }, + ], + expected: { + name: 'default', + httpMethod: 'POST', + path: 'webhook', + }, + }, + ]; + test.each(tests)('$description', ({ name, webhooks, expected }) => { + nodeType.description.webhooks = webhooks; + nodeTypes.getByNameAndVersion.mockReturnValueOnce(nodeType); + + const result = getWebhookDescription(name, workflow, node); + + expect(result).toEqual(expected); + expect(nodeTypes.getByNameAndVersion).toHaveBeenCalled(); + }); + }); + + describe('getNodeWebhookUrl', () => { + const webhookBaseUrl = 'http://localhost:5678/webhook'; + const webhookTestBaseUrl = 'http://localhost:5678/webhook-test'; + const additionalData = mock({ + webhookBaseUrl, + webhookTestBaseUrl, + }); + const webhookDescription = mock({ + name: 'default', + isFullPath: false, + }); + + const tests: Array<{ + description: string; + webhookPath: string | undefined; + webhookId: string | undefined; + isTest: boolean; + expected: string; + }> = [ + { + description: 'should return webhook URL with path', + webhookPath: 'webhook', + webhookId: undefined, + isTest: false, + expected: `${webhookBaseUrl}/workflow-id/test-node/webhook`, + }, + { + description: 'should handle path starting with /', + webhookPath: '/webhook', + webhookId: undefined, + isTest: false, + expected: `${webhookBaseUrl}/workflow-id/test-node/webhook`, + }, + { + description: 'should return webhook URL with webhookId', + webhookPath: 'webhook', + webhookId: 'abc123', + isTest: false, + expected: `${webhookBaseUrl}/abc123/webhook`, + }, + { + description: 'should return test webhook URL for test webhooks', + webhookPath: 'webhook', + webhookId: undefined, + isTest: true, + expected: `${webhookTestBaseUrl}/workflow-id/test-node/webhook`, + }, + ]; + test.each(tests)('$description', ({ webhookPath, webhookId, isTest, expected }) => { + node.webhookId = webhookId; + if (webhookPath) webhookDescription.path = webhookPath; + nodeType.description.webhooks = [webhookDescription]; + nodeTypes.getByNameAndVersion.mockReturnValueOnce(nodeType); + + expression.getSimpleParameterValue.mockImplementation((_node, parameterValue) => { + if (parameterValue === 'webhook') return webhookPath; + return parameterValue; + }); + + const result = getNodeWebhookUrl( + 'default', + workflow, + node, + additionalData, + 'manual', + {}, + isTest, + ); + + expect(result).toEqual(expected); + expect(expression.getSimpleParameterValue).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/construct-execution-metadata.ts b/packages/core/src/execution-engine/node-execution-context/utils/construct-execution-metadata.ts new file mode 100644 index 0000000000..31fb60a8ad --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/construct-execution-metadata.ts @@ -0,0 +1,17 @@ +import type { INodeExecutionData, IPairedItemData, NodeExecutionWithMetadata } from 'n8n-workflow'; + +/** + * Takes generic input data and brings it into the new json, pairedItem format n8n uses. + * @param {(IPairedItemData)} itemData + * @param {(INodeExecutionData[])} inputData + */ +export function constructExecutionMetaData( + inputData: INodeExecutionData[], + options: { itemData: IPairedItemData | IPairedItemData[] }, +): NodeExecutionWithMetadata[] { + const { itemData } = options; + return inputData.map((data: INodeExecutionData) => { + const { json, ...rest } = data; + return { json, pairedItem: itemData, ...rest } as NodeExecutionWithMetadata; + }); +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/copy-input-items.ts b/packages/core/src/execution-engine/node-execution-context/utils/copy-input-items.ts new file mode 100644 index 0000000000..c0808a379d --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/copy-input-items.ts @@ -0,0 +1,20 @@ +import type { INodeExecutionData, IDataObject } from 'n8n-workflow'; +import { deepCopy } from 'n8n-workflow'; + +/** + * Returns a copy of the items which only contains the json data and + * of that only the defined properties + */ +export function copyInputItems(items: INodeExecutionData[], properties: string[]): IDataObject[] { + return items.map((item) => { + const newItem: IDataObject = {}; + for (const property of properties) { + if (item.json[property] === undefined) { + newItem[property] = null; + } else { + newItem[property] = deepCopy(item.json[property]); + } + } + return newItem; + }); +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts b/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts index 9f76d0e106..d599d72841 100644 --- a/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/get-input-connection-data.ts @@ -20,9 +20,9 @@ import { } from 'n8n-workflow'; import { createNodeAsTool } from './create-node-as-tool'; -// eslint-disable-next-line import/no-cycle -import { SupplyDataContext } from '../../node-execution-context'; import type { ExecuteContext, WebhookContext } from '../../node-execution-context'; +// eslint-disable-next-line import/no-cycle +import { SupplyDataContext } from '../../node-execution-context/supply-data-context'; export async function getInputConnectionData( this: ExecuteContext | WebhookContext | SupplyDataContext, diff --git a/packages/core/src/execution-engine/node-execution-context/utils/normalize-items.ts b/packages/core/src/execution-engine/node-execution-context/utils/normalize-items.ts new file mode 100644 index 0000000000..cafc9f6c42 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/normalize-items.ts @@ -0,0 +1,47 @@ +import type { INodeExecutionData, IDataObject } from 'n8n-workflow'; +import { ApplicationError } from 'n8n-workflow'; + +/** + * Automatically put the objects under a 'json' key and don't error, + * if some objects contain json/binary keys and others don't, throws error 'Inconsistent item format' + * + * @param {INodeExecutionData | INodeExecutionData[]} executionData + */ +export function normalizeItems( + executionData: INodeExecutionData | INodeExecutionData[], +): INodeExecutionData[] { + if (typeof executionData === 'object' && !Array.isArray(executionData)) { + executionData = executionData.json ? [executionData] : [{ json: executionData as IDataObject }]; + } + + if (executionData.every((item) => typeof item === 'object' && 'json' in item)) + return executionData; + + if (executionData.some((item) => typeof item === 'object' && 'json' in item)) { + throw new ApplicationError('Inconsistent item format'); + } + + if (executionData.every((item) => typeof item === 'object' && 'binary' in item)) { + const normalizedItems: INodeExecutionData[] = []; + executionData.forEach((item) => { + const json = Object.keys(item).reduce((acc, key) => { + if (key === 'binary') return acc; + return { ...acc, [key]: item[key] }; + }, {}); + + normalizedItems.push({ + json, + binary: item.binary, + }); + }); + return normalizedItems; + } + + if (executionData.some((item) => typeof item === 'object' && 'binary' in item)) { + throw new ApplicationError('Inconsistent item format'); + } + + return executionData.map((item) => { + return { json: item }; + }); +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts b/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts deleted file mode 100644 index fb80eff52d..0000000000 --- a/packages/core/src/execution-engine/node-execution-context/utils/parse-request-object.ts +++ /dev/null @@ -1,468 +0,0 @@ -import { Container } from '@n8n/di'; -import type { AxiosHeaders, AxiosRequestConfig } from 'axios'; -import crypto from 'crypto'; -import FormData from 'form-data'; -import { Agent, type AgentOptions } from 'https'; -import type { GenericValue, IRequestOptions } from 'n8n-workflow'; -import { stringify } from 'qs'; -import { URL } from 'url'; - -import { Logger } from '@/logging/logger'; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const pushFormDataValue = (form: FormData, key: string, value: any) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - if (value?.hasOwnProperty('value') && value.hasOwnProperty('options')) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument - form.append(key, value.value, value.options); - } else { - form.append(key, value); - } -}; - -const createFormDataObject = (data: Record) => { - const formData = new FormData(); - const keys = Object.keys(data); - keys.forEach((key) => { - const formField = data[key]; - - if (formField instanceof Array) { - formField.forEach((item) => { - pushFormDataValue(formData, key, item); - }); - } else { - pushFormDataValue(formData, key, formField); - } - }); - return formData; -}; - -function searchForHeader(config: AxiosRequestConfig, headerName: string) { - if (config.headers === undefined) { - return undefined; - } - - const headerNames = Object.keys(config.headers); - headerName = headerName.toLowerCase(); - return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName); -} - -async function generateContentLengthHeader(config: AxiosRequestConfig) { - if (!(config.data instanceof FormData)) { - return; - } - try { - const length = await new Promise((res, rej) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - config.data.getLength((error: Error | null, dataLength: number) => { - if (error) rej(error); - else res(dataLength); - }); - }); - config.headers = { - ...config.headers, - 'content-length': length, - }; - } catch (error) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - Container.get(Logger).error('Unable to calculate form data length', { error }); - } -} - -const getHostFromRequestObject = ( - requestObject: Partial<{ - url: string; - uri: string; - baseURL: string; - }>, -): string | null => { - try { - const url = (requestObject.url ?? requestObject.uri) as string; - return new URL(url, requestObject.baseURL).hostname; - } catch (error) { - return null; - } -}; - -const getBeforeRedirectFn = - (agentOptions: AgentOptions, axiosConfig: AxiosRequestConfig) => - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (redirectedRequest: Record) => { - const redirectAgent = new Agent({ - ...agentOptions, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - servername: redirectedRequest.hostname, - }); - redirectedRequest.agent = redirectAgent; - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - redirectedRequest.agents.https = redirectAgent; - - if (axiosConfig.headers?.Authorization) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - redirectedRequest.headers.Authorization = axiosConfig.headers.Authorization; - } - if (axiosConfig.auth) { - redirectedRequest.auth = `${axiosConfig.auth.username}:${axiosConfig.auth.password}`; - } - }; - -/** - * This function is a temporary implementation that translates all http requests - * done via the request library to axios directly. - * We are not using n8n's interface as it would an unnecessary step, - * considering the `request` helper has been be deprecated and should be removed. - * @deprecated This is only used by legacy request helpers, that are also deprecated - */ -// eslint-disable-next-line complexity -export async function parseRequestObject(requestObject: IRequestOptions) { - const axiosConfig: AxiosRequestConfig = {}; - - if (requestObject.headers !== undefined) { - axiosConfig.headers = requestObject.headers as AxiosHeaders; - } - - // Let's start parsing the hardest part, which is the request body. - // The process here is as following? - // - Check if we have a `content-type` header. If this was set, - // we will follow - // - Check if the `form` property was set. If yes, then it's x-www-form-urlencoded - // - Check if the `formData` property exists. If yes, then it's multipart/form-data - // - Lastly, we should have a regular `body` that is probably a JSON. - - const contentTypeHeaderKeyName = - axiosConfig.headers && - Object.keys(axiosConfig.headers).find( - (headerName) => headerName.toLowerCase() === 'content-type', - ); - const contentType = - contentTypeHeaderKeyName && - (axiosConfig.headers?.[contentTypeHeaderKeyName] as string | undefined); - if (contentType === 'application/x-www-form-urlencoded' && requestObject.formData === undefined) { - // there are nodes incorrectly created, informing the content type header - // and also using formData. Request lib takes precedence for the formData. - // We will do the same. - // Merge body and form properties. - if (typeof requestObject.body === 'string') { - axiosConfig.data = requestObject.body; - } else { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - const allData = Object.assign(requestObject.body || {}, requestObject.form || {}) as Record< - string, - string - >; - if (requestObject.useQuerystring === true) { - axiosConfig.data = stringify(allData, { arrayFormat: 'repeat' }); - } else { - axiosConfig.data = stringify(allData); - } - } - } else if (contentType?.includes('multipart/form-data')) { - if (requestObject.formData !== undefined && requestObject.formData instanceof FormData) { - axiosConfig.data = requestObject.formData; - } else { - const allData: Partial = { - ...(requestObject.body as object | undefined), - ...(requestObject.formData as object | undefined), - }; - - axiosConfig.data = createFormDataObject(allData); - } - // replace the existing header with a new one that - // contains the boundary property. - delete axiosConfig.headers?.[contentTypeHeaderKeyName!]; - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - const headers = axiosConfig.data.getHeaders(); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); - await generateContentLengthHeader(axiosConfig); - } else { - // When using the `form` property it means the content should be x-www-form-urlencoded. - if (requestObject.form !== undefined && requestObject.body === undefined) { - // If we have only form - axiosConfig.data = - typeof requestObject.form === 'string' - ? stringify(requestObject.form, { format: 'RFC3986' }) - : stringify(requestObject.form).toString(); - if (axiosConfig.headers !== undefined) { - const headerName = searchForHeader(axiosConfig, 'content-type'); - if (headerName) { - delete axiosConfig.headers[headerName]; - } - axiosConfig.headers['Content-Type'] = 'application/x-www-form-urlencoded'; - } else { - axiosConfig.headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - }; - } - } else if (requestObject.formData !== undefined) { - // remove any "content-type" that might exist. - if (axiosConfig.headers !== undefined) { - const headers = Object.keys(axiosConfig.headers); - headers.forEach((header) => { - if (header.toLowerCase() === 'content-type') { - delete axiosConfig.headers?.[header]; - } - }); - } - - if (requestObject.formData instanceof FormData) { - axiosConfig.data = requestObject.formData; - } else { - axiosConfig.data = createFormDataObject(requestObject.formData as Record); - } - // Mix in headers as FormData creates the boundary. - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const headers = axiosConfig.data.getHeaders(); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); - await generateContentLengthHeader(axiosConfig); - } else if (requestObject.body !== undefined) { - // If we have body and possibly form - if (requestObject.form !== undefined && requestObject.body) { - // merge both objects when exist. - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - requestObject.body = Object.assign(requestObject.body, requestObject.form); - } - axiosConfig.data = requestObject.body as FormData | GenericValue | GenericValue[]; - } - } - - if (requestObject.uri !== undefined) { - axiosConfig.url = requestObject.uri?.toString(); - } - - if (requestObject.url !== undefined) { - axiosConfig.url = requestObject.url?.toString(); - } - - if (requestObject.baseURL !== undefined) { - axiosConfig.baseURL = requestObject.baseURL?.toString(); - } - - if (requestObject.method !== undefined) { - axiosConfig.method = requestObject.method; - } - - if (requestObject.qs !== undefined && Object.keys(requestObject.qs as object).length > 0) { - axiosConfig.params = requestObject.qs; - } - - function hasArrayFormatOptions( - arg: IRequestOptions, - ): arg is Required> { - if ( - typeof arg.qsStringifyOptions === 'object' && - arg.qsStringifyOptions !== null && - !Array.isArray(arg.qsStringifyOptions) && - 'arrayFormat' in arg.qsStringifyOptions - ) { - return true; - } - - return false; - } - - if ( - requestObject.useQuerystring === true || - (hasArrayFormatOptions(requestObject) && - requestObject.qsStringifyOptions.arrayFormat === 'repeat') - ) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'repeat' }); - }; - } else if (requestObject.useQuerystring === false) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'indices' }); - }; - } - - if ( - hasArrayFormatOptions(requestObject) && - requestObject.qsStringifyOptions.arrayFormat === 'brackets' - ) { - axiosConfig.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: 'brackets' }); - }; - } - - if (requestObject.auth !== undefined) { - // Check support for sendImmediately - if (requestObject.auth.bearer !== undefined) { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - Authorization: `Bearer ${requestObject.auth.bearer}`, - }); - } else { - const authObj = requestObject.auth; - // Request accepts both user/username and pass/password - axiosConfig.auth = { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - username: (authObj.user || authObj.username) as string, - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - password: (authObj.password || authObj.pass) as string, - }; - } - } - - // Only set header if we have a body, otherwise it may fail - if (requestObject.json === true) { - // Add application/json headers - do not set charset as it breaks a lot of stuff - // only add if no other accept headers was sent. - const acceptHeaderExists = - axiosConfig.headers === undefined - ? false - : Object.keys(axiosConfig.headers) - .map((headerKey) => headerKey.toLowerCase()) - .includes('accept'); - if (!acceptHeaderExists) { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - Accept: 'application/json', - }); - } - } - if (requestObject.json === false || requestObject.json === undefined) { - // Prevent json parsing - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - axiosConfig.transformResponse = (res) => res; - } - - // Axios will follow redirects by default, so we simply tell it otherwise if needed. - const { method } = requestObject; - if ( - (requestObject.followRedirect !== false && - (!method || method === 'GET' || method === 'HEAD')) || - requestObject.followAllRedirects - ) { - axiosConfig.maxRedirects = requestObject.maxRedirects; - } else { - axiosConfig.maxRedirects = 0; - } - - const host = getHostFromRequestObject(requestObject); - const agentOptions: AgentOptions = { ...requestObject.agentOptions }; - if (host) { - agentOptions.servername = host; - } - if (requestObject.rejectUnauthorized === false) { - agentOptions.rejectUnauthorized = false; - agentOptions.secureOptions = crypto.constants.SSL_OP_LEGACY_SERVER_CONNECT; - } - - axiosConfig.httpsAgent = new Agent(agentOptions); - - axiosConfig.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosConfig); - - if (requestObject.timeout !== undefined) { - axiosConfig.timeout = requestObject.timeout; - } - - if (requestObject.proxy !== undefined) { - // try our best to parse the url provided. - if (typeof requestObject.proxy === 'string') { - try { - const url = new URL(requestObject.proxy); - // eslint-disable-next-line @typescript-eslint/no-shadow - const host = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; - axiosConfig.proxy = { - host, - port: parseInt(url.port, 10), - protocol: url.protocol, - }; - if (!url.port) { - // Sets port to a default if not informed - if (url.protocol === 'http') { - axiosConfig.proxy.port = 80; - } else if (url.protocol === 'https') { - axiosConfig.proxy.port = 443; - } - } - if (url.username || url.password) { - axiosConfig.proxy.auth = { - username: url.username, - password: url.password, - }; - } - } catch (error) { - // Not a valid URL. We will try to simply parse stuff - // such as user:pass@host:port without protocol (we'll assume http) - if (requestObject.proxy.includes('@')) { - const [userpass, hostport] = requestObject.proxy.split('@'); - const [username, password] = userpass.split(':'); - const [hostname, port] = hostport.split(':'); - // eslint-disable-next-line @typescript-eslint/no-shadow - const host = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname; - axiosConfig.proxy = { - host, - port: parseInt(port, 10), - protocol: 'http', - auth: { - username, - password, - }, - }; - } else if (requestObject.proxy.includes(':')) { - const [hostname, port] = requestObject.proxy.split(':'); - axiosConfig.proxy = { - host: hostname, - port: parseInt(port, 10), - protocol: 'http', - }; - } else { - axiosConfig.proxy = { - host: requestObject.proxy, - port: 80, - protocol: 'http', - }; - } - } - } else { - axiosConfig.proxy = requestObject.proxy; - } - } - - if (requestObject.useStream) { - axiosConfig.responseType = 'stream'; - } else if (requestObject.encoding === null) { - // When downloading files, return an arrayBuffer. - axiosConfig.responseType = 'arraybuffer'; - } - - // If we don't set an accept header - // Axios forces "application/json, text/plan, */*" - // Which causes some nodes like NextCloud to break - // as the service returns XML unless requested otherwise. - const allHeaders = axiosConfig.headers ? Object.keys(axiosConfig.headers) : []; - if (!allHeaders.some((headerKey) => headerKey.toLowerCase() === 'accept')) { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { accept: '*/*' }); - } - if ( - requestObject.json !== false && - axiosConfig.data !== undefined && - axiosConfig.data !== '' && - !(axiosConfig.data instanceof Buffer) && - !allHeaders.some((headerKey) => headerKey.toLowerCase() === 'content-type') - ) { - // Use default header for application/json - // If we don't specify this here, axios will add - // application/json; charset=utf-8 - // and this breaks a lot of stuff - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { - 'content-type': 'application/json', - }); - } - - if (requestObject.simple === false) { - axiosConfig.validateStatus = () => true; - } - - /** - * Missing properties: - * encoding (need testing) - * gzip (ignored - default already works) - * resolveWithFullResponse (implemented elsewhere) - */ - return axiosConfig; -} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts new file mode 100644 index 0000000000..04657029a9 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts @@ -0,0 +1,1743 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/prefer-nullish-coalescing */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-shadow */ +import type { + ClientOAuth2Options, + ClientOAuth2RequestObject, + ClientOAuth2TokenData, + OAuth2CredentialData, +} from '@n8n/client-oauth2'; +import { ClientOAuth2 } from '@n8n/client-oauth2'; +import { Container } from '@n8n/di'; +import type { AxiosError, AxiosHeaders, AxiosRequestConfig, AxiosResponse } from 'axios'; +import axios from 'axios'; +import crypto, { createHmac } from 'crypto'; +import FormData from 'form-data'; +import { IncomingMessage } from 'http'; +import { Agent, type AgentOptions } from 'https'; +import get from 'lodash/get'; +import isEmpty from 'lodash/isEmpty'; +import merge from 'lodash/merge'; +import pick from 'lodash/pick'; +import type { + GenericValue, + IAdditionalCredentialOptions, + IAllExecuteFunctions, + ICredentialDataDecryptedObject, + IDataObject, + IExecuteData, + IExecuteFunctions, + IHttpRequestOptions, + IN8nHttpFullResponse, + IN8nHttpResponse, + INode, + INodeExecutionData, + IOAuth2Options, + IRequestOptions, + IRunExecutionData, + IWorkflowDataProxyAdditionalKeys, + IWorkflowExecuteAdditionalData, + NodeParameterValueType, + PaginationOptions, + RequestHelperFunctions, + Workflow, + WorkflowExecuteMode, +} from 'n8n-workflow'; +import { + NodeApiError, + NodeOperationError, + NodeSslError, + isObjectEmpty, + ExecutionBaseError, + jsonParse, + ApplicationError, + sleep, +} from 'n8n-workflow'; +import type { Token } from 'oauth-1.0a'; +import clientOAuth1 from 'oauth-1.0a'; +import { stringify } from 'qs'; +import { Readable } from 'stream'; +import url, { URL, URLSearchParams } from 'url'; + +import type { IResponseError } from '@/interfaces'; +import { Logger } from '@/logging/logger'; + +import { binaryToString } from './binary-helper-functions'; +import { parseIncomingMessage } from './parse-incoming-message'; + +axios.defaults.timeout = 300000; +// Prevent axios from adding x-form-www-urlencoded headers by default +axios.defaults.headers.post = {}; +axios.defaults.headers.put = {}; +axios.defaults.headers.patch = {}; +axios.defaults.paramsSerializer = (params) => { + if (params instanceof URLSearchParams) { + return params.toString(); + } + return stringify(params, { arrayFormat: 'indices' }); +}; +axios.interceptors.request.use((config) => { + // If no content-type is set by us, prevent axios from force-setting the content-type to `application/x-www-form-urlencoded` + if (config.data === undefined) { + config.headers.setContentType(false, false); + } + return config; +}); + +const validateUrl = (url?: string): boolean => { + if (!url) return false; + + try { + new URL(url); + return true; + } catch (error) { + return false; + } +}; + +function searchForHeader(config: AxiosRequestConfig, headerName: string) { + if (config.headers === undefined) { + return undefined; + } + + const headerNames = Object.keys(config.headers); + headerName = headerName.toLowerCase(); + return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName); +} + +const getHostFromRequestObject = ( + requestObject: Partial<{ + url: string; + uri: string; + baseURL: string; + }>, +): string | null => { + try { + const url = (requestObject.url ?? requestObject.uri) as string; + return new URL(url, requestObject.baseURL).hostname; + } catch (error) { + return null; + } +}; + +const getBeforeRedirectFn = + (agentOptions: AgentOptions, axiosConfig: AxiosRequestConfig) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (redirectedRequest: Record) => { + const redirectAgent = new Agent({ + ...agentOptions, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + servername: redirectedRequest.hostname, + }); + redirectedRequest.agent = redirectAgent; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + redirectedRequest.agents.https = redirectAgent; + + if (axiosConfig.headers?.Authorization) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + redirectedRequest.headers.Authorization = axiosConfig.headers.Authorization; + } + if (axiosConfig.auth) { + redirectedRequest.auth = `${axiosConfig.auth.username}:${axiosConfig.auth.password}`; + } + }; + +function digestAuthAxiosConfig( + axiosConfig: AxiosRequestConfig, + response: AxiosResponse, + auth: AxiosRequestConfig['auth'], +): AxiosRequestConfig { + const authDetails = response.headers['www-authenticate'] + .split(',') + .map((v: string) => v.split('=')); + if (authDetails) { + const nonceCount = '000000001'; + const cnonce = crypto.randomBytes(24).toString('hex'); + const realm: string = authDetails + .find((el: any) => el[0].toLowerCase().indexOf('realm') > -1)[1] + .replace(/"/g, ''); + // If authDetails does not have opaque, we should not add it to authorization. + const opaqueKV = authDetails.find((el: any) => el[0].toLowerCase().indexOf('opaque') > -1); + const opaque: string = opaqueKV ? opaqueKV[1].replace(/"/g, '') : undefined; + const nonce: string = authDetails + .find((el: any) => el[0].toLowerCase().indexOf('nonce') > -1)[1] + .replace(/"/g, ''); + const ha1 = crypto + .createHash('md5') + .update(`${auth?.username as string}:${realm}:${auth?.password as string}`) + .digest('hex'); + const urlURL = new url.URL(axios.getUri(axiosConfig)); + const path = urlURL.pathname + urlURL.search; + const ha2 = crypto + .createHash('md5') + .update(`${axiosConfig.method ?? 'GET'}:${path}`) + .digest('hex'); + const response = crypto + .createHash('md5') + .update(`${ha1}:${nonce}:${nonceCount}:${cnonce}:auth:${ha2}`) + .digest('hex'); + let authorization = + `Digest username="${auth?.username as string}",realm="${realm}",` + + `nonce="${nonce}",uri="${path}",qop="auth",algorithm="MD5",` + + `response="${response}",nc="${nonceCount}",cnonce="${cnonce}"`; + // Only when opaque exists, add it to authorization. + if (opaque) { + authorization += `,opaque="${opaque}"`; + } + if (axiosConfig.headers) { + axiosConfig.headers.authorization = authorization; + } else { + axiosConfig.headers = { authorization }; + } + } + return axiosConfig; +} + +export async function invokeAxios( + axiosConfig: AxiosRequestConfig, + authOptions: IRequestOptions['auth'] = {}, +) { + try { + return await axios(axiosConfig); + } catch (error) { + if (authOptions.sendImmediately !== false || !(error instanceof axios.AxiosError)) throw error; + // for digest-auth + const { response } = error; + if (response?.status !== 401 || !response.headers['www-authenticate']?.includes('nonce')) { + throw error; + } + const { auth } = axiosConfig; + delete axiosConfig.auth; + axiosConfig = digestAuthAxiosConfig(axiosConfig, response, auth); + return await axios(axiosConfig); + } +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const pushFormDataValue = (form: FormData, key: string, value: any) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + if (value?.hasOwnProperty('value') && value.hasOwnProperty('options')) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument + form.append(key, value.value, value.options); + } else { + form.append(key, value); + } +}; + +export const createFormDataObject = (data: Record) => { + const formData = new FormData(); + const keys = Object.keys(data); + keys.forEach((key) => { + const formField = data[key]; + + if (formField instanceof Array) { + formField.forEach((item) => { + pushFormDataValue(formData, key, item); + }); + } else { + pushFormDataValue(formData, key, formField); + } + }); + return formData; +}; + +async function generateContentLengthHeader(config: AxiosRequestConfig) { + if (!(config.data instanceof FormData)) { + return; + } + try { + const length = await new Promise((res, rej) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + config.data.getLength((error: Error | null, dataLength: number) => { + if (error) rej(error); + else res(dataLength); + }); + }); + config.headers = { + ...config.headers, + 'content-length': length, + }; + } catch (error) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + Container.get(Logger).error('Unable to calculate form data length', { error }); + } +} + +/** + * This function is a temporary implementation that translates all http requests + * done via the request library to axios directly. + * We are not using n8n's interface as it would an unnecessary step, + * considering the `request` helper has been be deprecated and should be removed. + * @deprecated This is only used by legacy request helpers, that are also deprecated + */ +// eslint-disable-next-line complexity +export async function parseRequestObject(requestObject: IRequestOptions) { + const axiosConfig: AxiosRequestConfig = {}; + + if (requestObject.headers !== undefined) { + axiosConfig.headers = requestObject.headers as AxiosHeaders; + } + + // Let's start parsing the hardest part, which is the request body. + // The process here is as following? + // - Check if we have a `content-type` header. If this was set, + // we will follow + // - Check if the `form` property was set. If yes, then it's x-www-form-urlencoded + // - Check if the `formData` property exists. If yes, then it's multipart/form-data + // - Lastly, we should have a regular `body` that is probably a JSON. + + const contentTypeHeaderKeyName = + axiosConfig.headers && + Object.keys(axiosConfig.headers).find( + (headerName) => headerName.toLowerCase() === 'content-type', + ); + const contentType = + contentTypeHeaderKeyName && + (axiosConfig.headers?.[contentTypeHeaderKeyName] as string | undefined); + if (contentType === 'application/x-www-form-urlencoded' && requestObject.formData === undefined) { + // there are nodes incorrectly created, informing the content type header + // and also using formData. Request lib takes precedence for the formData. + // We will do the same. + // Merge body and form properties. + if (typeof requestObject.body === 'string') { + axiosConfig.data = requestObject.body; + } else { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + const allData = Object.assign(requestObject.body || {}, requestObject.form || {}) as Record< + string, + string + >; + if (requestObject.useQuerystring === true) { + axiosConfig.data = stringify(allData, { arrayFormat: 'repeat' }); + } else { + axiosConfig.data = stringify(allData); + } + } + } else if (contentType?.includes('multipart/form-data')) { + if (requestObject.formData !== undefined && requestObject.formData instanceof FormData) { + axiosConfig.data = requestObject.formData; + } else { + const allData: Partial = { + ...(requestObject.body as object | undefined), + ...(requestObject.formData as object | undefined), + }; + + axiosConfig.data = createFormDataObject(allData); + } + // replace the existing header with a new one that + // contains the boundary property. + delete axiosConfig.headers?.[contentTypeHeaderKeyName!]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + const headers = axiosConfig.data.getHeaders(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); + await generateContentLengthHeader(axiosConfig); + } else { + // When using the `form` property it means the content should be x-www-form-urlencoded. + if (requestObject.form !== undefined && requestObject.body === undefined) { + // If we have only form + axiosConfig.data = + typeof requestObject.form === 'string' + ? stringify(requestObject.form, { format: 'RFC3986' }) + : stringify(requestObject.form).toString(); + if (axiosConfig.headers !== undefined) { + const headerName = searchForHeader(axiosConfig, 'content-type'); + if (headerName) { + delete axiosConfig.headers[headerName]; + } + axiosConfig.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } else { + axiosConfig.headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + }; + } + } else if (requestObject.formData !== undefined) { + // remove any "content-type" that might exist. + if (axiosConfig.headers !== undefined) { + const headers = Object.keys(axiosConfig.headers); + headers.forEach((header) => { + if (header.toLowerCase() === 'content-type') { + delete axiosConfig.headers?.[header]; + } + }); + } + + if (requestObject.formData instanceof FormData) { + axiosConfig.data = requestObject.formData; + } else { + axiosConfig.data = createFormDataObject(requestObject.formData as Record); + } + // Mix in headers as FormData creates the boundary. + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + const headers = axiosConfig.data.getHeaders(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers); + await generateContentLengthHeader(axiosConfig); + } else if (requestObject.body !== undefined) { + // If we have body and possibly form + if (requestObject.form !== undefined && requestObject.body) { + // merge both objects when exist. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + requestObject.body = Object.assign(requestObject.body, requestObject.form); + } + axiosConfig.data = requestObject.body as FormData | GenericValue | GenericValue[]; + } + } + + if (requestObject.uri !== undefined) { + axiosConfig.url = requestObject.uri?.toString(); + } + + if (requestObject.url !== undefined) { + axiosConfig.url = requestObject.url?.toString(); + } + + if (requestObject.baseURL !== undefined) { + axiosConfig.baseURL = requestObject.baseURL?.toString(); + } + + if (requestObject.method !== undefined) { + axiosConfig.method = requestObject.method; + } + + if (requestObject.qs !== undefined && Object.keys(requestObject.qs as object).length > 0) { + axiosConfig.params = requestObject.qs; + } + + function hasArrayFormatOptions( + arg: IRequestOptions, + ): arg is Required> { + if ( + typeof arg.qsStringifyOptions === 'object' && + arg.qsStringifyOptions !== null && + !Array.isArray(arg.qsStringifyOptions) && + 'arrayFormat' in arg.qsStringifyOptions + ) { + return true; + } + + return false; + } + + if ( + requestObject.useQuerystring === true || + (hasArrayFormatOptions(requestObject) && + requestObject.qsStringifyOptions.arrayFormat === 'repeat') + ) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'repeat' }); + }; + } else if (requestObject.useQuerystring === false) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'indices' }); + }; + } + + if ( + hasArrayFormatOptions(requestObject) && + requestObject.qsStringifyOptions.arrayFormat === 'brackets' + ) { + axiosConfig.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: 'brackets' }); + }; + } + + if (requestObject.auth !== undefined) { + // Check support for sendImmediately + if (requestObject.auth.bearer !== undefined) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + Authorization: `Bearer ${requestObject.auth.bearer}`, + }); + } else { + const authObj = requestObject.auth; + // Request accepts both user/username and pass/password + axiosConfig.auth = { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + username: (authObj.user || authObj.username) as string, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + password: (authObj.password || authObj.pass) as string, + }; + } + } + + // Only set header if we have a body, otherwise it may fail + if (requestObject.json === true) { + // Add application/json headers - do not set charset as it breaks a lot of stuff + // only add if no other accept headers was sent. + const acceptHeaderExists = + axiosConfig.headers === undefined + ? false + : Object.keys(axiosConfig.headers) + .map((headerKey) => headerKey.toLowerCase()) + .includes('accept'); + if (!acceptHeaderExists) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + Accept: 'application/json', + }); + } + } + if (requestObject.json === false || requestObject.json === undefined) { + // Prevent json parsing + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + axiosConfig.transformResponse = (res) => res; + } + + // Axios will follow redirects by default, so we simply tell it otherwise if needed. + const { method } = requestObject; + if ( + (requestObject.followRedirect !== false && + (!method || method === 'GET' || method === 'HEAD')) || + requestObject.followAllRedirects + ) { + axiosConfig.maxRedirects = requestObject.maxRedirects; + } else { + axiosConfig.maxRedirects = 0; + } + + const host = getHostFromRequestObject(requestObject); + const agentOptions: AgentOptions = { ...requestObject.agentOptions }; + if (host) { + agentOptions.servername = host; + } + if (requestObject.rejectUnauthorized === false) { + agentOptions.rejectUnauthorized = false; + agentOptions.secureOptions = crypto.constants.SSL_OP_LEGACY_SERVER_CONNECT; + } + + axiosConfig.httpsAgent = new Agent(agentOptions); + + axiosConfig.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosConfig); + + if (requestObject.timeout !== undefined) { + axiosConfig.timeout = requestObject.timeout; + } + + if (requestObject.proxy !== undefined) { + // try our best to parse the url provided. + if (typeof requestObject.proxy === 'string') { + try { + const url = new URL(requestObject.proxy); + // eslint-disable-next-line @typescript-eslint/no-shadow + const host = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; + axiosConfig.proxy = { + host, + port: parseInt(url.port, 10), + protocol: url.protocol, + }; + if (!url.port) { + // Sets port to a default if not informed + if (url.protocol === 'http') { + axiosConfig.proxy.port = 80; + } else if (url.protocol === 'https') { + axiosConfig.proxy.port = 443; + } + } + if (url.username || url.password) { + axiosConfig.proxy.auth = { + username: url.username, + password: url.password, + }; + } + } catch (error) { + // Not a valid URL. We will try to simply parse stuff + // such as user:pass@host:port without protocol (we'll assume http) + if (requestObject.proxy.includes('@')) { + const [userpass, hostport] = requestObject.proxy.split('@'); + const [username, password] = userpass.split(':'); + const [hostname, port] = hostport.split(':'); + // eslint-disable-next-line @typescript-eslint/no-shadow + const host = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname; + axiosConfig.proxy = { + host, + port: parseInt(port, 10), + protocol: 'http', + auth: { + username, + password, + }, + }; + } else if (requestObject.proxy.includes(':')) { + const [hostname, port] = requestObject.proxy.split(':'); + axiosConfig.proxy = { + host: hostname, + port: parseInt(port, 10), + protocol: 'http', + }; + } else { + axiosConfig.proxy = { + host: requestObject.proxy, + port: 80, + protocol: 'http', + }; + } + } + } else { + axiosConfig.proxy = requestObject.proxy; + } + } + + if (requestObject.useStream) { + axiosConfig.responseType = 'stream'; + } else if (requestObject.encoding === null) { + // When downloading files, return an arrayBuffer. + axiosConfig.responseType = 'arraybuffer'; + } + + // If we don't set an accept header + // Axios forces "application/json, text/plan, */*" + // Which causes some nodes like NextCloud to break + // as the service returns XML unless requested otherwise. + const allHeaders = axiosConfig.headers ? Object.keys(axiosConfig.headers) : []; + if (!allHeaders.some((headerKey) => headerKey.toLowerCase() === 'accept')) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { accept: '*/*' }); + } + if ( + requestObject.json !== false && + axiosConfig.data !== undefined && + axiosConfig.data !== '' && + !(axiosConfig.data instanceof Buffer) && + !allHeaders.some((headerKey) => headerKey.toLowerCase() === 'content-type') + ) { + // Use default header for application/json + // If we don't specify this here, axios will add + // application/json; charset=utf-8 + // and this breaks a lot of stuff + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { + 'content-type': 'application/json', + }); + } + + if (requestObject.simple === false) { + axiosConfig.validateStatus = () => true; + } + + /** + * Missing properties: + * encoding (need testing) + * gzip (ignored - default already works) + * resolveWithFullResponse (implemented elsewhere) + */ + return axiosConfig; +} + +/** + * @deprecated This is only used by legacy request helpers, that are also deprecated + */ +export async function proxyRequestToAxios( + workflow: Workflow | undefined, + additionalData: IWorkflowExecuteAdditionalData | undefined, + node: INode | undefined, + uriOrObject: string | IRequestOptions, + options?: IRequestOptions, +): Promise { + let axiosConfig: AxiosRequestConfig = { + maxBodyLength: Infinity, + maxContentLength: Infinity, + }; + let configObject: IRequestOptions; + if (typeof uriOrObject === 'string') { + configObject = { uri: uriOrObject, ...options }; + } else { + configObject = uriOrObject ?? {}; + } + + axiosConfig = Object.assign(axiosConfig, await parseRequestObject(configObject)); + + try { + const response = await invokeAxios(axiosConfig, configObject.auth); + let body = response.data; + if (body instanceof IncomingMessage && axiosConfig.responseType === 'stream') { + parseIncomingMessage(body); + } else if (body === '') { + body = axiosConfig.responseType === 'arraybuffer' ? Buffer.alloc(0) : undefined; + } + await additionalData?.hooks?.runHook('nodeFetchedData', [workflow?.id, node]); + return configObject.resolveWithFullResponse + ? { + body, + headers: { ...response.headers }, + statusCode: response.status, + statusMessage: response.statusText, + request: response.request, + } + : body; + } catch (error) { + const { config, response } = error; + + // Axios hydrates the original error with more data. We extract them. + // https://github.com/axios/axios/blob/master/lib/core/enhanceError.js + // Note: `code` is ignored as it's an expected part of the errorData. + if (error.isAxiosError) { + error.config = error.request = undefined; + error.options = pick(config ?? {}, ['url', 'method', 'data', 'headers']); + if (response) { + Container.get(Logger).debug('Request proxied to Axios failed', { status: response.status }); + let responseData = response.data; + + if (Buffer.isBuffer(responseData) || responseData instanceof Readable) { + responseData = await binaryToString(responseData); + } + + if (configObject.simple === false) { + if (configObject.resolveWithFullResponse) { + return { + body: responseData, + headers: response.headers, + statusCode: response.status, + statusMessage: response.statusText, + }; + } else { + return responseData; + } + } + + error.message = `${response.status as number} - ${JSON.stringify(responseData)}`; + throw Object.assign(error, { + statusCode: response.status, + /** + * Axios adds `status` when serializing, causing `status` to be available only to the client. + * Hence we add it explicitly to allow the backend to use it when resolving expressions. + */ + status: response.status, + error: responseData, + response: pick(response, ['headers', 'status', 'statusText']), + }); + } else if ('rejectUnauthorized' in configObject && error.code?.includes('CERT')) { + throw new NodeSslError(error); + } + } + + throw error; + } +} + +// eslint-disable-next-line complexity +export function convertN8nRequestToAxios(n8nRequest: IHttpRequestOptions): AxiosRequestConfig { + // Destructure properties with the same name first. + const { headers, method, timeout, auth, proxy, url } = n8nRequest; + + const axiosRequest: AxiosRequestConfig = { + headers: headers ?? {}, + method, + timeout, + auth, + proxy, + url, + maxBodyLength: Infinity, + maxContentLength: Infinity, + } as AxiosRequestConfig; + + axiosRequest.params = n8nRequest.qs; + + if (n8nRequest.abortSignal) { + axiosRequest.signal = n8nRequest.abortSignal; + } + + if (n8nRequest.baseURL !== undefined) { + axiosRequest.baseURL = n8nRequest.baseURL; + } + + if (n8nRequest.disableFollowRedirect === true) { + axiosRequest.maxRedirects = 0; + } + + if (n8nRequest.encoding !== undefined) { + axiosRequest.responseType = n8nRequest.encoding; + } + + const host = getHostFromRequestObject(n8nRequest); + const agentOptions: AgentOptions = {}; + if (host) { + agentOptions.servername = host; + } + if (n8nRequest.skipSslCertificateValidation === true) { + agentOptions.rejectUnauthorized = false; + } + axiosRequest.httpsAgent = new Agent(agentOptions); + + axiosRequest.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosRequest); + + if (n8nRequest.arrayFormat !== undefined) { + axiosRequest.paramsSerializer = (params) => { + return stringify(params, { arrayFormat: n8nRequest.arrayFormat }); + }; + } + + const { body } = n8nRequest; + if (body) { + // Let's add some useful header standards here. + const existingContentTypeHeaderKey = searchForHeader(axiosRequest, 'content-type'); + if (existingContentTypeHeaderKey === undefined) { + axiosRequest.headers = axiosRequest.headers || {}; + // We are only setting content type headers if the user did + // not set it already manually. We're not overriding, even if it's wrong. + if (body instanceof FormData) { + axiosRequest.headers = { + ...axiosRequest.headers, + ...body.getHeaders(), + }; + } else if (body instanceof URLSearchParams) { + axiosRequest.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } + } else if ( + axiosRequest.headers?.[existingContentTypeHeaderKey] === 'application/x-www-form-urlencoded' + ) { + axiosRequest.data = new URLSearchParams(n8nRequest.body as Record); + } + // if there is a body and it's empty (does not have properties), + // make sure not to send anything in it as some services fail when + // sending GET request with empty body. + if (typeof body === 'string' || (typeof body === 'object' && !isObjectEmpty(body))) { + axiosRequest.data = body; + } + } + + if (n8nRequest.json) { + const key = searchForHeader(axiosRequest, 'accept'); + // If key exists, then the user has set both accept + // header and the json flag. Header should take precedence. + if (!key) { + axiosRequest.headers = { + ...axiosRequest.headers, + Accept: 'application/json', + }; + } + } + + const userAgentHeader = searchForHeader(axiosRequest, 'user-agent'); + // If key exists, then the user has set both accept + // header and the json flag. Header should take precedence. + if (!userAgentHeader) { + axiosRequest.headers = { + ...axiosRequest.headers, + 'User-Agent': 'n8n', + }; + } + + if (n8nRequest.ignoreHttpStatusErrors) { + axiosRequest.validateStatus = () => true; + } + + return axiosRequest; +} + +const NoBodyHttpMethods = ['GET', 'HEAD', 'OPTIONS']; + +/** Remove empty request body on GET, HEAD, and OPTIONS requests */ +export const removeEmptyBody = (requestOptions: IHttpRequestOptions | IRequestOptions) => { + const method = requestOptions.method || 'GET'; + if (NoBodyHttpMethods.includes(method) && isEmpty(requestOptions.body)) { + delete requestOptions.body; + } +}; + +export async function httpRequest( + requestOptions: IHttpRequestOptions, +): Promise { + removeEmptyBody(requestOptions); + + const axiosRequest = convertN8nRequestToAxios(requestOptions); + if ( + axiosRequest.data === undefined || + (axiosRequest.method !== undefined && axiosRequest.method.toUpperCase() === 'GET') + ) { + delete axiosRequest.data; + } + + const result = await invokeAxios(axiosRequest, requestOptions.auth); + + if (requestOptions.returnFullResponse) { + return { + body: result.data, + headers: result.headers, + statusCode: result.status, + statusMessage: result.statusText, + }; + } + + return result.data; +} + +export function applyPaginationRequestData( + requestData: IRequestOptions, + paginationRequestData: PaginationOptions['request'], +): IRequestOptions { + const preparedPaginationData: Partial = { + ...paginationRequestData, + uri: paginationRequestData.url, + }; + + if ('formData' in requestData) { + preparedPaginationData.formData = paginationRequestData.body; + delete preparedPaginationData.body; + } else if ('form' in requestData) { + preparedPaginationData.form = paginationRequestData.body; + delete preparedPaginationData.body; + } + + return merge({}, requestData, preparedPaginationData); +} + +/** @deprecated make these requests using httpRequestWithAuthentication */ +export async function requestOAuth2( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IHttpRequestOptions | IRequestOptions, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + oAuth2Options?: IOAuth2Options, + isN8nRequest = false, +) { + removeEmptyBody(requestOptions); + + const credentials = (await this.getCredentials( + credentialsType, + )) as unknown as OAuth2CredentialData; + + // Only the OAuth2 with authorization code grant needs connection + if (credentials.grantType === 'authorizationCode' && credentials.oauthTokenData === undefined) { + throw new ApplicationError('OAuth credentials not connected'); + } + + const oAuthClient = new ClientOAuth2({ + clientId: credentials.clientId, + clientSecret: credentials.clientSecret, + accessTokenUri: credentials.accessTokenUrl, + scopes: (credentials.scope as string).split(' '), + ignoreSSLIssues: credentials.ignoreSSLIssues, + authentication: credentials.authentication ?? 'header', + }); + + let oauthTokenData = credentials.oauthTokenData as ClientOAuth2TokenData; + // if it's the first time using the credentials, get the access token and save it into the DB. + if ( + credentials.grantType === 'clientCredentials' && + (oauthTokenData === undefined || + Object.keys(oauthTokenData).length === 0 || + oauthTokenData.access_token === '') // stub + ) { + const { data } = await oAuthClient.credentials.getToken(); + // Find the credentials + if (!node.credentials?.[credentialsType]) { + throw new ApplicationError('Node does not have credential type', { + extra: { nodeName: node.name }, + tags: { credentialType: credentialsType }, + }); + } + + const nodeCredentials = node.credentials[credentialsType]; + credentials.oauthTokenData = data; + + // Save the refreshed token + await additionalData.credentialsHelper.updateCredentials( + nodeCredentials, + credentialsType, + credentials as unknown as ICredentialDataDecryptedObject, + ); + + oauthTokenData = data; + } + + const accessToken = + get(oauthTokenData, oAuth2Options?.property as string) || oauthTokenData.accessToken; + const refreshToken = oauthTokenData.refreshToken; + const token = oAuthClient.createToken( + { + ...oauthTokenData, + ...(accessToken ? { access_token: accessToken } : {}), + ...(refreshToken ? { refresh_token: refreshToken } : {}), + }, + oAuth2Options?.tokenType || oauthTokenData.tokenType, + ); + + (requestOptions as IRequestOptions).rejectUnauthorized = !credentials.ignoreSSLIssues; + + // Signs the request by adding authorization headers or query parameters depending + // on the token-type used. + const newRequestOptions = token.sign(requestOptions as ClientOAuth2RequestObject); + const newRequestHeaders = (newRequestOptions.headers = newRequestOptions.headers ?? {}); + // If keep bearer is false remove the it from the authorization header + if (oAuth2Options?.keepBearer === false && typeof newRequestHeaders.Authorization === 'string') { + newRequestHeaders.Authorization = newRequestHeaders.Authorization.split(' ')[1]; + } + if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { + Object.assign(newRequestHeaders, { + [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, + }); + } + if (isN8nRequest) { + return await this.helpers.httpRequest(newRequestOptions).catch(async (error: AxiosError) => { + if (error.response?.status === 401) { + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" expired. Should revalidate.`, + ); + const tokenRefreshOptions: IDataObject = {}; + if (oAuth2Options?.includeCredentialsOnRefreshOnBody) { + const body: IDataObject = { + client_id: credentials.clientId, + ...(credentials.grantType === 'authorizationCode' && { + client_secret: credentials.clientSecret as string, + }), + }; + tokenRefreshOptions.body = body; + tokenRefreshOptions.headers = { + Authorization: '', + }; + } + + let newToken; + + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, + ); + // if it's OAuth2 with client credentials grant type, get a new token + // instead of refreshing it. + if (credentials.grantType === 'clientCredentials') { + newToken = await token.client.credentials.getToken(); + } else { + newToken = await token.refresh(tokenRefreshOptions as unknown as ClientOAuth2Options); + } + + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, + ); + + credentials.oauthTokenData = newToken.data; + // Find the credentials + if (!node.credentials?.[credentialsType]) { + throw new ApplicationError('Node does not have credential type', { + extra: { nodeName: node.name, credentialType: credentialsType }, + }); + } + const nodeCredentials = node.credentials[credentialsType]; + await additionalData.credentialsHelper.updateCredentials( + nodeCredentials, + credentialsType, + credentials as unknown as ICredentialDataDecryptedObject, + ); + const refreshedRequestOption = newToken.sign(requestOptions as ClientOAuth2RequestObject); + + if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { + Object.assign(newRequestHeaders, { + [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, + }); + } + + return await this.helpers.httpRequest(refreshedRequestOption); + } + throw error; + }); + } + const tokenExpiredStatusCode = + oAuth2Options?.tokenExpiredStatusCode === undefined + ? 401 + : oAuth2Options?.tokenExpiredStatusCode; + + return await this.helpers + .request(newRequestOptions as IRequestOptions) + .then((response) => { + const requestOptions = newRequestOptions as any; + if ( + requestOptions.resolveWithFullResponse === true && + requestOptions.simple === false && + response.statusCode === tokenExpiredStatusCode + ) { + throw response; + } + return response; + }) + .catch(async (error: IResponseError) => { + if (error.statusCode === tokenExpiredStatusCode) { + // Token is probably not valid anymore. So try refresh it. + const tokenRefreshOptions: IDataObject = {}; + if (oAuth2Options?.includeCredentialsOnRefreshOnBody) { + const body: IDataObject = { + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + }; + tokenRefreshOptions.body = body; + // Override authorization property so the credentials are not included in it + tokenRefreshOptions.headers = { + Authorization: '', + }; + } + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" expired. Should revalidate.`, + ); + + let newToken; + + // if it's OAuth2 with client credentials grant type, get a new token + // instead of refreshing it. + if (credentials.grantType === 'clientCredentials') { + newToken = await token.client.credentials.getToken(); + } else { + newToken = await token.refresh(tokenRefreshOptions as unknown as ClientOAuth2Options); + } + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, + ); + + credentials.oauthTokenData = newToken.data; + + // Find the credentials + if (!node.credentials?.[credentialsType]) { + throw new ApplicationError('Node does not have credential type', { + tags: { credentialType: credentialsType }, + extra: { nodeName: node.name }, + }); + } + const nodeCredentials = node.credentials[credentialsType]; + + // Save the refreshed token + await additionalData.credentialsHelper.updateCredentials( + nodeCredentials, + credentialsType, + credentials as unknown as ICredentialDataDecryptedObject, + ); + + this.logger.debug( + `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been saved to database successfully.`, + ); + + // Make the request again with the new token + const newRequestOptions = newToken.sign(requestOptions as ClientOAuth2RequestObject); + newRequestOptions.headers = newRequestOptions.headers ?? {}; + + if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { + Object.assign(newRequestOptions.headers, { + [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, + }); + } + + return await this.helpers.request(newRequestOptions as IRequestOptions); + } + + // Unknown error so simply throw it + throw error; + }); +} + +/** @deprecated make these requests using httpRequestWithAuthentication */ +export async function requestOAuth1( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IHttpRequestOptions | IRequestOptions, + isN8nRequest = false, +) { + removeEmptyBody(requestOptions); + + const credentials = await this.getCredentials(credentialsType); + + if (credentials === undefined) { + throw new ApplicationError('No credentials were returned'); + } + + if (credentials.oauthTokenData === undefined) { + throw new ApplicationError('OAuth credentials not connected'); + } + + const oauth = new clientOAuth1({ + consumer: { + key: credentials.consumerKey as string, + secret: credentials.consumerSecret as string, + }, + signature_method: credentials.signatureMethod as string, + hash_function(base, key) { + let algorithm: string; + switch (credentials.signatureMethod) { + case 'HMAC-SHA256': + algorithm = 'sha256'; + break; + case 'HMAC-SHA512': + algorithm = 'sha512'; + break; + default: + algorithm = 'sha1'; + break; + } + return createHmac(algorithm, key).update(base).digest('base64'); + }, + }); + + const oauthTokenData = credentials.oauthTokenData as IDataObject; + + const token: Token = { + key: oauthTokenData.oauth_token as string, + secret: oauthTokenData.oauth_token_secret as string, + }; + + // @ts-expect-error @TECH_DEBT: Remove request library + requestOptions.data = { ...requestOptions.qs, ...requestOptions.form }; + + // Fixes issue that OAuth1 library only works with "url" property and not with "uri" + if ('uri' in requestOptions && !requestOptions.url) { + requestOptions.url = requestOptions.uri; + delete requestOptions.uri; + } + + requestOptions.headers = oauth.toHeader( + oauth.authorize(requestOptions as unknown as clientOAuth1.RequestOptions, token), + ) as unknown as Record; + if (isN8nRequest) { + return await this.helpers.httpRequest(requestOptions as IHttpRequestOptions); + } + + return await this.helpers + .request(requestOptions as IRequestOptions) + .catch(async (error: IResponseError) => { + // Unknown error so simply throw it + throw error; + }); +} + +export async function httpRequestWithAuthentication( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IHttpRequestOptions, + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + additionalCredentialOptions?: IAdditionalCredentialOptions, +) { + removeEmptyBody(requestOptions); + + // Cancel this request on execution cancellation + if ('getExecutionCancelSignal' in this) { + requestOptions.abortSignal = this.getExecutionCancelSignal(); + } + + let credentialsDecrypted: ICredentialDataDecryptedObject | undefined; + try { + const parentTypes = additionalData.credentialsHelper.getParentTypes(credentialsType); + + if (parentTypes.includes('oAuth1Api')) { + return await requestOAuth1.call(this, credentialsType, requestOptions, true); + } + if (parentTypes.includes('oAuth2Api')) { + return await requestOAuth2.call( + this, + credentialsType, + requestOptions, + node, + additionalData, + additionalCredentialOptions?.oauth2, + true, + ); + } + + if (additionalCredentialOptions?.credentialsDecrypted) { + credentialsDecrypted = additionalCredentialOptions.credentialsDecrypted.data; + } else { + credentialsDecrypted = + await this.getCredentials(credentialsType); + } + + if (credentialsDecrypted === undefined) { + throw new NodeOperationError( + node, + `Node "${node.name}" does not have any credentials of type "${credentialsType}" set`, + { level: 'warning' }, + ); + } + + const data = await additionalData.credentialsHelper.preAuthentication( + { helpers: this.helpers }, + credentialsDecrypted, + credentialsType, + node, + false, + ); + + if (data) { + // make the updated property in the credentials + // available to the authenticate method + Object.assign(credentialsDecrypted, data); + } + + requestOptions = await additionalData.credentialsHelper.authenticate( + credentialsDecrypted, + credentialsType, + requestOptions, + workflow, + node, + ); + return await httpRequest(requestOptions); + } catch (error) { + // if there is a pre authorization method defined and + // the method failed due to unauthorized request + if ( + error.response?.status === 401 && + additionalData.credentialsHelper.preAuthentication !== undefined + ) { + try { + if (credentialsDecrypted !== undefined) { + // try to refresh the credentials + const data = await additionalData.credentialsHelper.preAuthentication( + { helpers: this.helpers }, + credentialsDecrypted, + credentialsType, + node, + true, + ); + + if (data) { + // make the updated property in the credentials + // available to the authenticate method + Object.assign(credentialsDecrypted, data); + } + + requestOptions = await additionalData.credentialsHelper.authenticate( + credentialsDecrypted, + credentialsType, + requestOptions, + workflow, + node, + ); + } + // retry the request + return await httpRequest(requestOptions); + } catch (error) { + throw new NodeApiError(this.getNode(), error); + } + } + + throw new NodeApiError(this.getNode(), error); + } +} + +/** @deprecated use httpRequestWithAuthentication */ +export async function requestWithAuthentication( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IRequestOptions, + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + additionalCredentialOptions?: IAdditionalCredentialOptions, + itemIndex?: number, +) { + removeEmptyBody(requestOptions); + + let credentialsDecrypted: ICredentialDataDecryptedObject | undefined; + + try { + const parentTypes = additionalData.credentialsHelper.getParentTypes(credentialsType); + + if (credentialsType === 'oAuth1Api' || parentTypes.includes('oAuth1Api')) { + return await requestOAuth1.call(this, credentialsType, requestOptions, false); + } + if (credentialsType === 'oAuth2Api' || parentTypes.includes('oAuth2Api')) { + return await requestOAuth2.call( + this, + credentialsType, + requestOptions, + node, + additionalData, + additionalCredentialOptions?.oauth2, + false, + ); + } + + if (additionalCredentialOptions?.credentialsDecrypted) { + credentialsDecrypted = additionalCredentialOptions.credentialsDecrypted.data; + } else { + credentialsDecrypted = await this.getCredentials( + credentialsType, + itemIndex, + ); + } + + if (credentialsDecrypted === undefined) { + throw new NodeOperationError( + node, + `Node "${node.name}" does not have any credentials of type "${credentialsType}" set`, + { level: 'warning' }, + ); + } + + const data = await additionalData.credentialsHelper.preAuthentication( + { helpers: this.helpers }, + credentialsDecrypted, + credentialsType, + node, + false, + ); + + if (data) { + // make the updated property in the credentials + // available to the authenticate method + Object.assign(credentialsDecrypted, data); + } + + requestOptions = (await additionalData.credentialsHelper.authenticate( + credentialsDecrypted, + credentialsType, + requestOptions as IHttpRequestOptions, + workflow, + node, + )) as IRequestOptions; + return await proxyRequestToAxios(workflow, additionalData, node, requestOptions); + } catch (error) { + try { + if (credentialsDecrypted !== undefined) { + // try to refresh the credentials + const data = await additionalData.credentialsHelper.preAuthentication( + { helpers: this.helpers }, + credentialsDecrypted, + credentialsType, + node, + true, + ); + + if (data) { + // make the updated property in the credentials + // available to the authenticate method + Object.assign(credentialsDecrypted, data); + requestOptions = (await additionalData.credentialsHelper.authenticate( + credentialsDecrypted, + credentialsType, + requestOptions as IHttpRequestOptions, + workflow, + node, + )) as IRequestOptions; + // retry the request + return await proxyRequestToAxios(workflow, additionalData, node, requestOptions); + } + } + throw error; + } catch (error) { + if (error instanceof ExecutionBaseError) throw error; + + throw new NodeApiError(this.getNode(), error); + } + } +} + +export const getRequestHelperFunctions = ( + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + runExecutionData: IRunExecutionData | null = null, + connectionInputData: INodeExecutionData[] = [], +): RequestHelperFunctions => { + const getResolvedValue = ( + parameterValue: NodeParameterValueType, + itemIndex: number, + runIndex: number, + executeData: IExecuteData, + additionalKeys?: IWorkflowDataProxyAdditionalKeys, + returnObjectAsString = false, + ): NodeParameterValueType => { + const mode: WorkflowExecuteMode = 'internal'; + + if ( + typeof parameterValue === 'object' || + (typeof parameterValue === 'string' && parameterValue.charAt(0) === '=') + ) { + return workflow.expression.getParameterValue( + parameterValue, + runExecutionData, + runIndex, + itemIndex, + node.name, + connectionInputData, + mode, + additionalKeys ?? {}, + executeData, + returnObjectAsString, + ); + } + + return parameterValue; + }; + + // eslint-disable-next-line complexity + async function requestWithAuthenticationPaginated( + this: IExecuteFunctions, + requestOptions: IRequestOptions, + itemIndex: number, + paginationOptions: PaginationOptions, + credentialsType?: string, + additionalCredentialOptions?: IAdditionalCredentialOptions, + ): Promise { + const responseData = []; + if (!requestOptions.qs) { + requestOptions.qs = {}; + } + requestOptions.resolveWithFullResponse = true; + requestOptions.simple = false; + + let tempResponseData: IN8nHttpFullResponse; + let makeAdditionalRequest: boolean; + let paginateRequestData: PaginationOptions['request']; + + const runIndex = 0; + + const additionalKeys: IWorkflowDataProxyAdditionalKeys = { + $request: requestOptions, + $response: {} as IN8nHttpFullResponse, + $version: node.typeVersion, + $pageCount: 0, + }; + + const executeData: IExecuteData = { + data: {}, + node, + source: null, + }; + + const hashData = { + identicalCount: 0, + previousLength: 0, + previousHash: '', + }; + do { + paginateRequestData = getResolvedValue( + paginationOptions.request as unknown as NodeParameterValueType, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as object as PaginationOptions['request']; + + const tempRequestOptions = applyPaginationRequestData(requestOptions, paginateRequestData); + + if (!validateUrl(tempRequestOptions.uri as string)) { + throw new NodeOperationError(node, `'${paginateRequestData.url}' is not a valid URL.`, { + itemIndex, + runIndex, + type: 'invalid_url', + }); + } + + if (credentialsType) { + tempResponseData = await this.helpers.requestWithAuthentication.call( + this, + credentialsType, + tempRequestOptions, + additionalCredentialOptions, + ); + } else { + tempResponseData = await this.helpers.request(tempRequestOptions); + } + + const newResponse: IN8nHttpFullResponse = Object.assign( + { + body: {}, + headers: {}, + statusCode: 0, + }, + pick(tempResponseData, ['body', 'headers', 'statusCode']), + ); + + let contentBody: Exclude; + + if (newResponse.body instanceof Readable && paginationOptions.binaryResult !== true) { + // Keep the original string version that we can use it to hash if needed + contentBody = await binaryToString(newResponse.body as Buffer | Readable); + + const responseContentType = newResponse.headers['content-type']?.toString() ?? ''; + if (responseContentType.includes('application/json')) { + newResponse.body = jsonParse(contentBody, { fallbackValue: {} }); + } else { + newResponse.body = contentBody; + } + tempResponseData.__bodyResolved = true; + tempResponseData.body = newResponse.body; + } else { + contentBody = newResponse.body; + } + + if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) { + // If the data is not binary (and so not a stream), or an etag is present, + // we check via etag or hash if identical data is received + + let contentLength = 0; + if ('content-length' in tempResponseData.headers) { + contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0; + } + + if (hashData.previousLength === contentLength) { + let hash: string; + if (tempResponseData.headers.etag) { + // If an etag is provided, we use it as "hash" + hash = tempResponseData.headers.etag as string; + } else { + // If there is no etag, we calculate a hash from the data in the body + if (typeof contentBody !== 'string') { + contentBody = JSON.stringify(contentBody); + } + hash = crypto.createHash('md5').update(contentBody).digest('base64'); + } + + if (hashData.previousHash === hash) { + hashData.identicalCount += 1; + if (hashData.identicalCount > 2) { + // Length was identical 5x and hash 3x + throw new NodeOperationError( + node, + 'The returned response was identical 5x, so requests got stopped', + { + itemIndex, + description: + 'Check if "Pagination Completed When" has been configured correctly.', + }, + ); + } + } else { + hashData.identicalCount = 0; + } + hashData.previousHash = hash; + } else { + hashData.identicalCount = 0; + } + hashData.previousLength = contentLength; + } + + responseData.push(tempResponseData); + + additionalKeys.$response = newResponse; + additionalKeys.$pageCount = (additionalKeys.$pageCount ?? 0) + 1; + + const maxRequests = getResolvedValue( + paginationOptions.maxRequests, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as number; + + if (maxRequests && additionalKeys.$pageCount >= maxRequests) { + break; + } + + makeAdditionalRequest = getResolvedValue( + paginationOptions.continue, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as boolean; + + if (makeAdditionalRequest) { + if (paginationOptions.requestInterval) { + const requestInterval = getResolvedValue( + paginationOptions.requestInterval, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as number; + + await sleep(requestInterval); + } + if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) { + // We have it configured to let all requests pass no matter the response code + // via "requestOptions.simple = false" to not by default fail if it is for example + // configured to stop on 404 response codes. For that reason we have to throw here + // now an error manually if the response code is not a success one. + let data = tempResponseData.body; + if (data instanceof Readable && paginationOptions.binaryResult !== true) { + data = await binaryToString(data as Buffer | Readable); + } else if (typeof data === 'object') { + data = JSON.stringify(data); + } + + throw Object.assign(new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`), { + statusCode: tempResponseData.statusCode, + error: data, + isAxiosError: true, + response: { + headers: tempResponseData.headers, + status: tempResponseData.statusCode, + statusText: tempResponseData.statusMessage, + }, + }); + } + } + } while (makeAdditionalRequest); + + return responseData; + } + + return { + httpRequest, + requestWithAuthenticationPaginated, + async httpRequestWithAuthentication( + this, + credentialsType, + requestOptions, + additionalCredentialOptions, + ): Promise { + return await httpRequestWithAuthentication.call( + this, + credentialsType, + requestOptions, + workflow, + node, + additionalData, + additionalCredentialOptions, + ); + }, + + request: async (uriOrObject, options) => + await proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options), + + async requestWithAuthentication( + this, + credentialsType, + requestOptions, + additionalCredentialOptions, + itemIndex, + ): Promise { + return await requestWithAuthentication.call( + this, + credentialsType, + requestOptions, + workflow, + node, + additionalData, + additionalCredentialOptions, + itemIndex, + ); + }, + + async requestOAuth1( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IRequestOptions, + ): Promise { + return await requestOAuth1.call(this, credentialsType, requestOptions); + }, + + async requestOAuth2( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: IRequestOptions, + oAuth2Options?: IOAuth2Options, + ): Promise { + return await requestOAuth2.call( + this, + credentialsType, + requestOptions, + node, + additionalData, + oAuth2Options, + ); + }, + }; +}; diff --git a/packages/core/src/execution-engine/node-execution-context/utils/return-json-array.ts b/packages/core/src/execution-engine/node-execution-context/utils/return-json-array.ts new file mode 100644 index 0000000000..d8d6b3be72 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/return-json-array.ts @@ -0,0 +1,25 @@ +import type { IDataObject, INodeExecutionData } from 'n8n-workflow'; + +/** + * Takes generic input data and brings it into the json format n8n uses. + * + * @param {(IDataObject | IDataObject[])} jsonData + */ +export function returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExecutionData[] { + const returnData: INodeExecutionData[] = []; + + if (!Array.isArray(jsonData)) { + jsonData = [jsonData]; + } + + jsonData.forEach((data: IDataObject & { json?: IDataObject }) => { + if (data?.json) { + // We already have the JSON key so avoid double wrapping + returnData.push({ ...data, json: data.json }); + } else { + returnData.push({ json: data }); + } + }); + + return returnData; +} diff --git a/packages/core/src/execution-engine/node-execution-context/utils/scheduling-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/scheduling-helper-functions.ts new file mode 100644 index 0000000000..d2e0034dc0 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/scheduling-helper-functions.ts @@ -0,0 +1,12 @@ +import { Container } from '@n8n/di'; +import type { SchedulingFunctions, Workflow } from 'n8n-workflow'; + +import { ScheduledTaskManager } from '../../scheduled-task-manager'; + +export const getSchedulingFunctions = (workflow: Workflow): SchedulingFunctions => { + const scheduledTaskManager = Container.get(ScheduledTaskManager); + return { + registerCron: (cronExpression, onTick) => + scheduledTaskManager.registerCron(workflow, cronExpression, onTick), + }; +}; diff --git a/packages/core/src/execution-engine/node-execution-context/utils/ssh-tunnel-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/ssh-tunnel-helper-functions.ts new file mode 100644 index 0000000000..daeadd2e9f --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/ssh-tunnel-helper-functions.ts @@ -0,0 +1,11 @@ +import { Container } from '@n8n/di'; +import type { SSHTunnelFunctions } from 'n8n-workflow'; + +import { SSHClientsManager } from '../../ssh-clients-manager'; + +export const getSSHTunnelFunctions = (): SSHTunnelFunctions => { + const sshClientsManager = Container.get(SSHClientsManager); + return { + getSSHClient: async (credentials) => await sshClientsManager.getClient(credentials), + }; +}; diff --git a/packages/core/src/execution-engine/node-execution-context/utils/webhook-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/webhook-helper-functions.ts new file mode 100644 index 0000000000..f1ee0edd1b --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/webhook-helper-functions.ts @@ -0,0 +1,67 @@ +import type { + WebhookType, + Workflow, + INode, + IWorkflowExecuteAdditionalData, + WorkflowExecuteMode, + IWorkflowDataProxyAdditionalKeys, + IWebhookDescription, +} from 'n8n-workflow'; +import { NodeHelpers } from 'n8n-workflow'; + +/** Returns the full webhook description of the webhook with the given name */ +export function getWebhookDescription( + name: WebhookType, + workflow: Workflow, + node: INode, +): IWebhookDescription | undefined { + const nodeType = workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion); + + // Node does not have any webhooks so return + if (nodeType.description.webhooks === undefined) return; + + for (const webhookDescription of nodeType.description.webhooks) { + if (webhookDescription.name === name) { + return webhookDescription; + } + } + + return undefined; +} + +/** Returns the webhook URL of the webhook with the given name */ +export function getNodeWebhookUrl( + name: WebhookType, + workflow: Workflow, + node: INode, + additionalData: IWorkflowExecuteAdditionalData, + mode: WorkflowExecuteMode, + additionalKeys: IWorkflowDataProxyAdditionalKeys, + isTest?: boolean, +): string | undefined { + let baseUrl = additionalData.webhookBaseUrl; + if (isTest === true) { + baseUrl = additionalData.webhookTestBaseUrl; + } + + const webhookDescription = getWebhookDescription(name, workflow, node); + if (webhookDescription === undefined) return; + + const path = workflow.expression.getSimpleParameterValue( + node, + webhookDescription.path, + mode, + additionalKeys, + ); + if (path === undefined) return; + + const isFullPath: boolean = workflow.expression.getSimpleParameterValue( + node, + webhookDescription.isFullPath, + mode, + additionalKeys, + undefined, + false, + ) as boolean; + return NodeHelpers.getNodeWebhookUrl(baseUrl, workflow.id, node, path.toString(), isFullPath); +} diff --git a/packages/core/src/execution-engine/node-execution-context/webhook-context.ts b/packages/core/src/execution-engine/node-execution-context/webhook-context.ts index dba5c8124c..73f7164753 100644 --- a/packages/core/src/execution-engine/node-execution-context/webhook-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/webhook-context.ts @@ -18,16 +18,12 @@ import type { } from 'n8n-workflow'; import { ApplicationError, createDeferredPromise } from 'n8n-workflow'; -// eslint-disable-next-line import/no-cycle -import { - getNodeWebhookUrl, - getRequestHelperFunctions, - returnJsonArray, -} from '@/node-execute-functions'; - import { NodeExecutionContext } from './node-execution-context'; import { copyBinaryFile, getBinaryHelperFunctions } from './utils/binary-helper-functions'; import { getInputConnectionData } from './utils/get-input-connection-data'; +import { getRequestHelperFunctions } from './utils/request-helper-functions'; +import { returnJsonArray } from './utils/return-json-array'; +import { getNodeWebhookUrl } from './utils/webhook-helper-functions'; export class WebhookContext extends NodeExecutionContext implements IWebhookFunctions { readonly helpers: IWebhookFunctions['helpers']; diff --git a/packages/core/src/node-execute-functions.ts b/packages/core/src/node-execute-functions.ts index 34a4757962..ebeea341ea 100644 --- a/packages/core/src/node-execute-functions.ts +++ b/packages/core/src/node-execute-functions.ts @@ -1,1541 +1,14 @@ -/* eslint-disable @typescript-eslint/no-unsafe-argument */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/prefer-nullish-coalescing */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-shadow */ import type { - ClientOAuth2Options, - ClientOAuth2RequestObject, - ClientOAuth2TokenData, - OAuth2CredentialData, -} from '@n8n/client-oauth2'; -import { ClientOAuth2 } from '@n8n/client-oauth2'; -import { Container } from '@n8n/di'; -import type { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'; -import axios from 'axios'; -import crypto, { createHmac } from 'crypto'; -import FormData from 'form-data'; -import { IncomingMessage } from 'http'; -import { Agent, type AgentOptions } from 'https'; -import get from 'lodash/get'; -import isEmpty from 'lodash/isEmpty'; -import merge from 'lodash/merge'; -import pick from 'lodash/pick'; -import type { - IAdditionalCredentialOptions, - IAllExecuteFunctions, - ICredentialDataDecryptedObject, - IDataObject, - IExecuteData, - IExecuteFunctions, - IHttpRequestOptions, - IN8nHttpFullResponse, - IN8nHttpResponse, INode, - INodeExecutionData, - IOAuth2Options, - IPairedItemData, IPollFunctions, - IRequestOptions, - IRunExecutionData, ITriggerFunctions, - IWebhookDescription, - IWorkflowDataProxyAdditionalKeys, IWorkflowExecuteAdditionalData, - NodeExecutionWithMetadata, - NodeParameterValueType, - PaginationOptions, - RequestHelperFunctions, Workflow, WorkflowActivateMode, WorkflowExecuteMode, - SSHTunnelFunctions, - WebhookType, - SchedulingFunctions, } from 'n8n-workflow'; -import { - NodeApiError, - NodeHelpers, - NodeOperationError, - NodeSslError, - deepCopy, - isObjectEmpty, - ExecutionBaseError, - jsonParse, - ApplicationError, - sleep, -} from 'n8n-workflow'; -import type { Token } from 'oauth-1.0a'; -import clientOAuth1 from 'oauth-1.0a'; -import { stringify } from 'qs'; -import { Readable } from 'stream'; -import url, { URL, URLSearchParams } from 'url'; -import { Logger } from '@/logging/logger'; - -// eslint-disable-next-line import/no-cycle -import { - binaryToString, - parseIncomingMessage, - parseRequestObject, - PollContext, - TriggerContext, -} from './execution-engine/node-execution-context'; -import { ScheduledTaskManager } from './execution-engine/scheduled-task-manager'; -import { SSHClientsManager } from './execution-engine/ssh-clients-manager'; -import type { IResponseError } from './interfaces'; - -axios.defaults.timeout = 300000; -// Prevent axios from adding x-form-www-urlencoded headers by default -axios.defaults.headers.post = {}; -axios.defaults.headers.put = {}; -axios.defaults.headers.patch = {}; -axios.defaults.paramsSerializer = (params) => { - if (params instanceof URLSearchParams) { - return params.toString(); - } - return stringify(params, { arrayFormat: 'indices' }); -}; -axios.interceptors.request.use((config) => { - // If no content-type is set by us, prevent axios from force-setting the content-type to `application/x-www-form-urlencoded` - if (config.data === undefined) { - config.headers.setContentType(false, false); - } - return config; -}); - -export const validateUrl = (url?: string): boolean => { - if (!url) return false; - - try { - new URL(url); - return true; - } catch (error) { - return false; - } -}; - -function searchForHeader(config: AxiosRequestConfig, headerName: string) { - if (config.headers === undefined) { - return undefined; - } - - const headerNames = Object.keys(config.headers); - headerName = headerName.toLowerCase(); - return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName); -} - -const getHostFromRequestObject = ( - requestObject: Partial<{ - url: string; - uri: string; - baseURL: string; - }>, -): string | null => { - try { - const url = (requestObject.url ?? requestObject.uri) as string; - return new URL(url, requestObject.baseURL).hostname; - } catch (error) { - return null; - } -}; - -const getBeforeRedirectFn = - (agentOptions: AgentOptions, axiosConfig: AxiosRequestConfig) => - (redirectedRequest: Record) => { - const redirectAgent = new Agent({ - ...agentOptions, - servername: redirectedRequest.hostname, - }); - redirectedRequest.agent = redirectAgent; - redirectedRequest.agents.https = redirectAgent; - - if (axiosConfig.headers?.Authorization) { - redirectedRequest.headers.Authorization = axiosConfig.headers.Authorization; - } - if (axiosConfig.auth) { - redirectedRequest.auth = `${axiosConfig.auth.username}:${axiosConfig.auth.password}`; - } - }; - -function digestAuthAxiosConfig( - axiosConfig: AxiosRequestConfig, - response: AxiosResponse, - auth: AxiosRequestConfig['auth'], -): AxiosRequestConfig { - const authDetails = response.headers['www-authenticate'] - .split(',') - .map((v: string) => v.split('=')); - if (authDetails) { - const nonceCount = '000000001'; - const cnonce = crypto.randomBytes(24).toString('hex'); - const realm: string = authDetails - .find((el: any) => el[0].toLowerCase().indexOf('realm') > -1)[1] - .replace(/"/g, ''); - // If authDetails does not have opaque, we should not add it to authorization. - const opaqueKV = authDetails.find((el: any) => el[0].toLowerCase().indexOf('opaque') > -1); - const opaque: string = opaqueKV ? opaqueKV[1].replace(/"/g, '') : undefined; - const nonce: string = authDetails - .find((el: any) => el[0].toLowerCase().indexOf('nonce') > -1)[1] - .replace(/"/g, ''); - const ha1 = crypto - .createHash('md5') - .update(`${auth?.username as string}:${realm}:${auth?.password as string}`) - .digest('hex'); - const urlURL = new url.URL(axios.getUri(axiosConfig)); - const path = urlURL.pathname + urlURL.search; - const ha2 = crypto - .createHash('md5') - .update(`${axiosConfig.method ?? 'GET'}:${path}`) - .digest('hex'); - const response = crypto - .createHash('md5') - .update(`${ha1}:${nonce}:${nonceCount}:${cnonce}:auth:${ha2}`) - .digest('hex'); - let authorization = - `Digest username="${auth?.username as string}",realm="${realm}",` + - `nonce="${nonce}",uri="${path}",qop="auth",algorithm="MD5",` + - `response="${response}",nc="${nonceCount}",cnonce="${cnonce}"`; - // Only when opaque exists, add it to authorization. - if (opaque) { - authorization += `,opaque="${opaque}"`; - } - if (axiosConfig.headers) { - axiosConfig.headers.authorization = authorization; - } else { - axiosConfig.headers = { authorization }; - } - } - return axiosConfig; -} - -export async function invokeAxios( - axiosConfig: AxiosRequestConfig, - authOptions: IRequestOptions['auth'] = {}, -) { - try { - return await axios(axiosConfig); - } catch (error) { - if (authOptions.sendImmediately !== false || !(error instanceof axios.AxiosError)) throw error; - // for digest-auth - const { response } = error; - if (response?.status !== 401 || !response.headers['www-authenticate']?.includes('nonce')) { - throw error; - } - const { auth } = axiosConfig; - delete axiosConfig.auth; - axiosConfig = digestAuthAxiosConfig(axiosConfig, response, auth); - return await axios(axiosConfig); - } -} - -/** - * @deprecated This is only used by legacy request helpers, that are also deprecated - */ -export async function proxyRequestToAxios( - workflow: Workflow | undefined, - additionalData: IWorkflowExecuteAdditionalData | undefined, - node: INode | undefined, - uriOrObject: string | IRequestOptions, - options?: IRequestOptions, -): Promise { - let axiosConfig: AxiosRequestConfig = { - maxBodyLength: Infinity, - maxContentLength: Infinity, - }; - let configObject: IRequestOptions; - if (typeof uriOrObject === 'string') { - configObject = { uri: uriOrObject, ...options }; - } else { - configObject = uriOrObject ?? {}; - } - - axiosConfig = Object.assign(axiosConfig, await parseRequestObject(configObject)); - - try { - const response = await invokeAxios(axiosConfig, configObject.auth); - let body = response.data; - if (body instanceof IncomingMessage && axiosConfig.responseType === 'stream') { - parseIncomingMessage(body); - } else if (body === '') { - body = axiosConfig.responseType === 'arraybuffer' ? Buffer.alloc(0) : undefined; - } - await additionalData?.hooks?.runHook('nodeFetchedData', [workflow?.id, node]); - return configObject.resolveWithFullResponse - ? { - body, - headers: { ...response.headers }, - statusCode: response.status, - statusMessage: response.statusText, - request: response.request, - } - : body; - } catch (error) { - const { config, response } = error; - - // Axios hydrates the original error with more data. We extract them. - // https://github.com/axios/axios/blob/master/lib/core/enhanceError.js - // Note: `code` is ignored as it's an expected part of the errorData. - if (error.isAxiosError) { - error.config = error.request = undefined; - error.options = pick(config ?? {}, ['url', 'method', 'data', 'headers']); - if (response) { - Container.get(Logger).debug('Request proxied to Axios failed', { status: response.status }); - let responseData = response.data; - - if (Buffer.isBuffer(responseData) || responseData instanceof Readable) { - responseData = await binaryToString(responseData); - } - - if (configObject.simple === false) { - if (configObject.resolveWithFullResponse) { - return { - body: responseData, - headers: response.headers, - statusCode: response.status, - statusMessage: response.statusText, - }; - } else { - return responseData; - } - } - - error.message = `${response.status as number} - ${JSON.stringify(responseData)}`; - throw Object.assign(error, { - statusCode: response.status, - /** - * Axios adds `status` when serializing, causing `status` to be available only to the client. - * Hence we add it explicitly to allow the backend to use it when resolving expressions. - */ - status: response.status, - error: responseData, - response: pick(response, ['headers', 'status', 'statusText']), - }); - } else if ('rejectUnauthorized' in configObject && error.code?.includes('CERT')) { - throw new NodeSslError(error); - } - } - - throw error; - } -} - -// eslint-disable-next-line complexity -function convertN8nRequestToAxios(n8nRequest: IHttpRequestOptions): AxiosRequestConfig { - // Destructure properties with the same name first. - const { headers, method, timeout, auth, proxy, url } = n8nRequest; - - const axiosRequest: AxiosRequestConfig = { - headers: headers ?? {}, - method, - timeout, - auth, - proxy, - url, - maxBodyLength: Infinity, - maxContentLength: Infinity, - } as AxiosRequestConfig; - - axiosRequest.params = n8nRequest.qs; - - if (n8nRequest.abortSignal) { - axiosRequest.signal = n8nRequest.abortSignal; - } - - if (n8nRequest.baseURL !== undefined) { - axiosRequest.baseURL = n8nRequest.baseURL; - } - - if (n8nRequest.disableFollowRedirect === true) { - axiosRequest.maxRedirects = 0; - } - - if (n8nRequest.encoding !== undefined) { - axiosRequest.responseType = n8nRequest.encoding; - } - - const host = getHostFromRequestObject(n8nRequest); - const agentOptions: AgentOptions = {}; - if (host) { - agentOptions.servername = host; - } - if (n8nRequest.skipSslCertificateValidation === true) { - agentOptions.rejectUnauthorized = false; - } - axiosRequest.httpsAgent = new Agent(agentOptions); - - axiosRequest.beforeRedirect = getBeforeRedirectFn(agentOptions, axiosRequest); - - if (n8nRequest.arrayFormat !== undefined) { - axiosRequest.paramsSerializer = (params) => { - return stringify(params, { arrayFormat: n8nRequest.arrayFormat }); - }; - } - - const { body } = n8nRequest; - if (body) { - // Let's add some useful header standards here. - const existingContentTypeHeaderKey = searchForHeader(axiosRequest, 'content-type'); - if (existingContentTypeHeaderKey === undefined) { - axiosRequest.headers = axiosRequest.headers || {}; - // We are only setting content type headers if the user did - // not set it already manually. We're not overriding, even if it's wrong. - if (body instanceof FormData) { - axiosRequest.headers = { - ...axiosRequest.headers, - ...body.getHeaders(), - }; - } else if (body instanceof URLSearchParams) { - axiosRequest.headers['Content-Type'] = 'application/x-www-form-urlencoded'; - } - } else if ( - axiosRequest.headers?.[existingContentTypeHeaderKey] === 'application/x-www-form-urlencoded' - ) { - axiosRequest.data = new URLSearchParams(n8nRequest.body as Record); - } - // if there is a body and it's empty (does not have properties), - // make sure not to send anything in it as some services fail when - // sending GET request with empty body. - if (typeof body === 'string' || (typeof body === 'object' && !isObjectEmpty(body))) { - axiosRequest.data = body; - } - } - - if (n8nRequest.json) { - const key = searchForHeader(axiosRequest, 'accept'); - // If key exists, then the user has set both accept - // header and the json flag. Header should take precedence. - if (!key) { - axiosRequest.headers = { - ...axiosRequest.headers, - Accept: 'application/json', - }; - } - } - - const userAgentHeader = searchForHeader(axiosRequest, 'user-agent'); - // If key exists, then the user has set both accept - // header and the json flag. Header should take precedence. - if (!userAgentHeader) { - axiosRequest.headers = { - ...axiosRequest.headers, - 'User-Agent': 'n8n', - }; - } - - if (n8nRequest.ignoreHttpStatusErrors) { - axiosRequest.validateStatus = () => true; - } - - return axiosRequest; -} - -const NoBodyHttpMethods = ['GET', 'HEAD', 'OPTIONS']; - -/** Remove empty request body on GET, HEAD, and OPTIONS requests */ -export const removeEmptyBody = (requestOptions: IHttpRequestOptions | IRequestOptions) => { - const method = requestOptions.method || 'GET'; - if (NoBodyHttpMethods.includes(method) && isEmpty(requestOptions.body)) { - delete requestOptions.body; - } -}; - -export async function httpRequest( - requestOptions: IHttpRequestOptions, -): Promise { - removeEmptyBody(requestOptions); - - const axiosRequest = convertN8nRequestToAxios(requestOptions); - if ( - axiosRequest.data === undefined || - (axiosRequest.method !== undefined && axiosRequest.method.toUpperCase() === 'GET') - ) { - delete axiosRequest.data; - } - - const result = await invokeAxios(axiosRequest, requestOptions.auth); - - if (requestOptions.returnFullResponse) { - return { - body: result.data, - headers: result.headers, - statusCode: result.status, - statusMessage: result.statusText, - }; - } - - return result.data; -} - -export function applyPaginationRequestData( - requestData: IRequestOptions, - paginationRequestData: PaginationOptions['request'], -): IRequestOptions { - const preparedPaginationData: Partial = { - ...paginationRequestData, - uri: paginationRequestData.url, - }; - - if ('formData' in requestData) { - preparedPaginationData.formData = paginationRequestData.body; - delete preparedPaginationData.body; - } else if ('form' in requestData) { - preparedPaginationData.form = paginationRequestData.body; - delete preparedPaginationData.body; - } - - return merge({}, requestData, preparedPaginationData); -} - -/** - * Makes a request using OAuth data for authentication - * - * @param {(IHttpRequestOptions | IRequestOptions)} requestOptions - * - */ -export async function requestOAuth2( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IHttpRequestOptions | IRequestOptions, - node: INode, - additionalData: IWorkflowExecuteAdditionalData, - oAuth2Options?: IOAuth2Options, - isN8nRequest = false, -) { - removeEmptyBody(requestOptions); - - const credentials = (await this.getCredentials( - credentialsType, - )) as unknown as OAuth2CredentialData; - - // Only the OAuth2 with authorization code grant needs connection - if (credentials.grantType === 'authorizationCode' && credentials.oauthTokenData === undefined) { - throw new ApplicationError('OAuth credentials not connected'); - } - - const oAuthClient = new ClientOAuth2({ - clientId: credentials.clientId, - clientSecret: credentials.clientSecret, - accessTokenUri: credentials.accessTokenUrl, - scopes: (credentials.scope as string).split(' '), - ignoreSSLIssues: credentials.ignoreSSLIssues, - authentication: credentials.authentication ?? 'header', - }); - - let oauthTokenData = credentials.oauthTokenData as ClientOAuth2TokenData; - // if it's the first time using the credentials, get the access token and save it into the DB. - if ( - credentials.grantType === 'clientCredentials' && - (oauthTokenData === undefined || - Object.keys(oauthTokenData).length === 0 || - oauthTokenData.access_token === '') // stub - ) { - const { data } = await oAuthClient.credentials.getToken(); - // Find the credentials - if (!node.credentials?.[credentialsType]) { - throw new ApplicationError('Node does not have credential type', { - extra: { nodeName: node.name }, - tags: { credentialType: credentialsType }, - }); - } - - const nodeCredentials = node.credentials[credentialsType]; - credentials.oauthTokenData = data; - - // Save the refreshed token - await additionalData.credentialsHelper.updateCredentials( - nodeCredentials, - credentialsType, - credentials as unknown as ICredentialDataDecryptedObject, - ); - - oauthTokenData = data; - } - - const accessToken = - get(oauthTokenData, oAuth2Options?.property as string) || oauthTokenData.accessToken; - const refreshToken = oauthTokenData.refreshToken; - const token = oAuthClient.createToken( - { - ...oauthTokenData, - ...(accessToken ? { access_token: accessToken } : {}), - ...(refreshToken ? { refresh_token: refreshToken } : {}), - }, - oAuth2Options?.tokenType || oauthTokenData.tokenType, - ); - - (requestOptions as IRequestOptions).rejectUnauthorized = !credentials.ignoreSSLIssues; - - // Signs the request by adding authorization headers or query parameters depending - // on the token-type used. - const newRequestOptions = token.sign(requestOptions as ClientOAuth2RequestObject); - const newRequestHeaders = (newRequestOptions.headers = newRequestOptions.headers ?? {}); - // If keep bearer is false remove the it from the authorization header - if (oAuth2Options?.keepBearer === false && typeof newRequestHeaders.Authorization === 'string') { - newRequestHeaders.Authorization = newRequestHeaders.Authorization.split(' ')[1]; - } - if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { - Object.assign(newRequestHeaders, { - [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, - }); - } - if (isN8nRequest) { - return await this.helpers.httpRequest(newRequestOptions).catch(async (error: AxiosError) => { - if (error.response?.status === 401) { - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" expired. Should revalidate.`, - ); - const tokenRefreshOptions: IDataObject = {}; - if (oAuth2Options?.includeCredentialsOnRefreshOnBody) { - const body: IDataObject = { - client_id: credentials.clientId, - ...(credentials.grantType === 'authorizationCode' && { - client_secret: credentials.clientSecret as string, - }), - }; - tokenRefreshOptions.body = body; - tokenRefreshOptions.headers = { - Authorization: '', - }; - } - - let newToken; - - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, - ); - // if it's OAuth2 with client credentials grant type, get a new token - // instead of refreshing it. - if (credentials.grantType === 'clientCredentials') { - newToken = await token.client.credentials.getToken(); - } else { - newToken = await token.refresh(tokenRefreshOptions as unknown as ClientOAuth2Options); - } - - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, - ); - - credentials.oauthTokenData = newToken.data; - // Find the credentials - if (!node.credentials?.[credentialsType]) { - throw new ApplicationError('Node does not have credential type', { - extra: { nodeName: node.name, credentialType: credentialsType }, - }); - } - const nodeCredentials = node.credentials[credentialsType]; - await additionalData.credentialsHelper.updateCredentials( - nodeCredentials, - credentialsType, - credentials as unknown as ICredentialDataDecryptedObject, - ); - const refreshedRequestOption = newToken.sign(requestOptions as ClientOAuth2RequestObject); - - if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { - Object.assign(newRequestHeaders, { - [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, - }); - } - - return await this.helpers.httpRequest(refreshedRequestOption); - } - throw error; - }); - } - const tokenExpiredStatusCode = - oAuth2Options?.tokenExpiredStatusCode === undefined - ? 401 - : oAuth2Options?.tokenExpiredStatusCode; - - return await this.helpers - .request(newRequestOptions as IRequestOptions) - .then((response) => { - const requestOptions = newRequestOptions as any; - if ( - requestOptions.resolveWithFullResponse === true && - requestOptions.simple === false && - response.statusCode === tokenExpiredStatusCode - ) { - throw response; - } - return response; - }) - .catch(async (error: IResponseError) => { - if (error.statusCode === tokenExpiredStatusCode) { - // Token is probably not valid anymore. So try refresh it. - const tokenRefreshOptions: IDataObject = {}; - if (oAuth2Options?.includeCredentialsOnRefreshOnBody) { - const body: IDataObject = { - client_id: credentials.clientId, - client_secret: credentials.clientSecret, - }; - tokenRefreshOptions.body = body; - // Override authorization property so the credentials are not included in it - tokenRefreshOptions.headers = { - Authorization: '', - }; - } - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" expired. Should revalidate.`, - ); - - let newToken; - - // if it's OAuth2 with client credentials grant type, get a new token - // instead of refreshing it. - if (credentials.grantType === 'clientCredentials') { - newToken = await token.client.credentials.getToken(); - } else { - newToken = await token.refresh(tokenRefreshOptions as unknown as ClientOAuth2Options); - } - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been renewed.`, - ); - - credentials.oauthTokenData = newToken.data; - - // Find the credentials - if (!node.credentials?.[credentialsType]) { - throw new ApplicationError('Node does not have credential type', { - tags: { credentialType: credentialsType }, - extra: { nodeName: node.name }, - }); - } - const nodeCredentials = node.credentials[credentialsType]; - - // Save the refreshed token - await additionalData.credentialsHelper.updateCredentials( - nodeCredentials, - credentialsType, - credentials as unknown as ICredentialDataDecryptedObject, - ); - - this.logger.debug( - `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been saved to database successfully.`, - ); - - // Make the request again with the new token - const newRequestOptions = newToken.sign(requestOptions as ClientOAuth2RequestObject); - newRequestOptions.headers = newRequestOptions.headers ?? {}; - - if (oAuth2Options?.keyToIncludeInAccessTokenHeader) { - Object.assign(newRequestOptions.headers, { - [oAuth2Options.keyToIncludeInAccessTokenHeader]: token.accessToken, - }); - } - - return await this.helpers.request(newRequestOptions as IRequestOptions); - } - - // Unknown error so simply throw it - throw error; - }); -} - -/** - * Makes a request using OAuth1 data for authentication - */ -export async function requestOAuth1( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IHttpRequestOptions | IRequestOptions, - isN8nRequest = false, -) { - removeEmptyBody(requestOptions); - - const credentials = await this.getCredentials(credentialsType); - - if (credentials === undefined) { - throw new ApplicationError('No credentials were returned'); - } - - if (credentials.oauthTokenData === undefined) { - throw new ApplicationError('OAuth credentials not connected'); - } - - const oauth = new clientOAuth1({ - consumer: { - key: credentials.consumerKey as string, - secret: credentials.consumerSecret as string, - }, - signature_method: credentials.signatureMethod as string, - hash_function(base, key) { - let algorithm: string; - switch (credentials.signatureMethod) { - case 'HMAC-SHA256': - algorithm = 'sha256'; - break; - case 'HMAC-SHA512': - algorithm = 'sha512'; - break; - default: - algorithm = 'sha1'; - break; - } - return createHmac(algorithm, key).update(base).digest('base64'); - }, - }); - - const oauthTokenData = credentials.oauthTokenData as IDataObject; - - const token: Token = { - key: oauthTokenData.oauth_token as string, - secret: oauthTokenData.oauth_token_secret as string, - }; - - // @ts-expect-error @TECH_DEBT: Remove request library - requestOptions.data = { ...requestOptions.qs, ...requestOptions.form }; - - // Fixes issue that OAuth1 library only works with "url" property and not with "uri" - if ('uri' in requestOptions && !requestOptions.url) { - requestOptions.url = requestOptions.uri; - delete requestOptions.uri; - } - - requestOptions.headers = oauth.toHeader( - oauth.authorize(requestOptions as unknown as clientOAuth1.RequestOptions, token), - ) as unknown as Record; - if (isN8nRequest) { - return await this.helpers.httpRequest(requestOptions as IHttpRequestOptions); - } - - return await this.helpers - .request(requestOptions as IRequestOptions) - .catch(async (error: IResponseError) => { - // Unknown error so simply throw it - throw error; - }); -} - -export async function httpRequestWithAuthentication( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IHttpRequestOptions, - workflow: Workflow, - node: INode, - additionalData: IWorkflowExecuteAdditionalData, - additionalCredentialOptions?: IAdditionalCredentialOptions, -) { - removeEmptyBody(requestOptions); - - // Cancel this request on execution cancellation - if ('getExecutionCancelSignal' in this) { - requestOptions.abortSignal = this.getExecutionCancelSignal(); - } - - let credentialsDecrypted: ICredentialDataDecryptedObject | undefined; - try { - const parentTypes = additionalData.credentialsHelper.getParentTypes(credentialsType); - - if (parentTypes.includes('oAuth1Api')) { - return await requestOAuth1.call(this, credentialsType, requestOptions, true); - } - if (parentTypes.includes('oAuth2Api')) { - return await requestOAuth2.call( - this, - credentialsType, - requestOptions, - node, - additionalData, - additionalCredentialOptions?.oauth2, - true, - ); - } - - if (additionalCredentialOptions?.credentialsDecrypted) { - credentialsDecrypted = additionalCredentialOptions.credentialsDecrypted.data; - } else { - credentialsDecrypted = - await this.getCredentials(credentialsType); - } - - if (credentialsDecrypted === undefined) { - throw new NodeOperationError( - node, - `Node "${node.name}" does not have any credentials of type "${credentialsType}" set`, - { level: 'warning' }, - ); - } - - const data = await additionalData.credentialsHelper.preAuthentication( - { helpers: this.helpers }, - credentialsDecrypted, - credentialsType, - node, - false, - ); - - if (data) { - // make the updated property in the credentials - // available to the authenticate method - Object.assign(credentialsDecrypted, data); - } - - requestOptions = await additionalData.credentialsHelper.authenticate( - credentialsDecrypted, - credentialsType, - requestOptions, - workflow, - node, - ); - return await httpRequest(requestOptions); - } catch (error) { - // if there is a pre authorization method defined and - // the method failed due to unauthorized request - if ( - error.response?.status === 401 && - additionalData.credentialsHelper.preAuthentication !== undefined - ) { - try { - if (credentialsDecrypted !== undefined) { - // try to refresh the credentials - const data = await additionalData.credentialsHelper.preAuthentication( - { helpers: this.helpers }, - credentialsDecrypted, - credentialsType, - node, - true, - ); - - if (data) { - // make the updated property in the credentials - // available to the authenticate method - Object.assign(credentialsDecrypted, data); - } - - requestOptions = await additionalData.credentialsHelper.authenticate( - credentialsDecrypted, - credentialsType, - requestOptions, - workflow, - node, - ); - } - // retry the request - return await httpRequest(requestOptions); - } catch (error) { - throw new NodeApiError(this.getNode(), error); - } - } - - throw new NodeApiError(this.getNode(), error); - } -} - -/** - * Takes generic input data and brings it into the json format n8n uses. - * - * @param {(IDataObject | IDataObject[])} jsonData - */ -export function returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExecutionData[] { - const returnData: INodeExecutionData[] = []; - - if (!Array.isArray(jsonData)) { - jsonData = [jsonData]; - } - - jsonData.forEach((data: IDataObject & { json?: IDataObject }) => { - if (data?.json) { - // We already have the JSON key so avoid double wrapping - returnData.push({ ...data, json: data.json }); - } else { - returnData.push({ json: data }); - } - }); - - return returnData; -} - -/** - * Takes generic input data and brings it into the new json, pairedItem format n8n uses. - * @param {(IPairedItemData)} itemData - * @param {(INodeExecutionData[])} inputData - */ -export function constructExecutionMetaData( - inputData: INodeExecutionData[], - options: { itemData: IPairedItemData | IPairedItemData[] }, -): NodeExecutionWithMetadata[] { - const { itemData } = options; - return inputData.map((data: INodeExecutionData) => { - const { json, ...rest } = data; - return { json, pairedItem: itemData, ...rest } as NodeExecutionWithMetadata; - }); -} - -/** - * Automatically put the objects under a 'json' key and don't error, - * if some objects contain json/binary keys and others don't, throws error 'Inconsistent item format' - * - * @param {INodeExecutionData | INodeExecutionData[]} executionData - */ -export function normalizeItems( - executionData: INodeExecutionData | INodeExecutionData[], -): INodeExecutionData[] { - if (typeof executionData === 'object' && !Array.isArray(executionData)) { - executionData = executionData.json ? [executionData] : [{ json: executionData as IDataObject }]; - } - - if (executionData.every((item) => typeof item === 'object' && 'json' in item)) - return executionData; - - if (executionData.some((item) => typeof item === 'object' && 'json' in item)) { - throw new ApplicationError('Inconsistent item format'); - } - - if (executionData.every((item) => typeof item === 'object' && 'binary' in item)) { - const normalizedItems: INodeExecutionData[] = []; - executionData.forEach((item) => { - const json = Object.keys(item).reduce((acc, key) => { - if (key === 'binary') return acc; - return { ...acc, [key]: item[key] }; - }, {}); - - normalizedItems.push({ - json, - binary: item.binary, - }); - }); - return normalizedItems; - } - - if (executionData.some((item) => typeof item === 'object' && 'binary' in item)) { - throw new ApplicationError('Inconsistent item format'); - } - - return executionData.map((item) => { - return { json: item }; - }); -} - -// TODO: Move up later -export async function requestWithAuthentication( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IRequestOptions, - workflow: Workflow, - node: INode, - additionalData: IWorkflowExecuteAdditionalData, - additionalCredentialOptions?: IAdditionalCredentialOptions, - itemIndex?: number, -) { - removeEmptyBody(requestOptions); - - let credentialsDecrypted: ICredentialDataDecryptedObject | undefined; - - try { - const parentTypes = additionalData.credentialsHelper.getParentTypes(credentialsType); - - if (credentialsType === 'oAuth1Api' || parentTypes.includes('oAuth1Api')) { - return await requestOAuth1.call(this, credentialsType, requestOptions, false); - } - if (credentialsType === 'oAuth2Api' || parentTypes.includes('oAuth2Api')) { - return await requestOAuth2.call( - this, - credentialsType, - requestOptions, - node, - additionalData, - additionalCredentialOptions?.oauth2, - false, - ); - } - - if (additionalCredentialOptions?.credentialsDecrypted) { - credentialsDecrypted = additionalCredentialOptions.credentialsDecrypted.data; - } else { - credentialsDecrypted = await this.getCredentials( - credentialsType, - itemIndex, - ); - } - - if (credentialsDecrypted === undefined) { - throw new NodeOperationError( - node, - `Node "${node.name}" does not have any credentials of type "${credentialsType}" set`, - { level: 'warning' }, - ); - } - - const data = await additionalData.credentialsHelper.preAuthentication( - { helpers: this.helpers }, - credentialsDecrypted, - credentialsType, - node, - false, - ); - - if (data) { - // make the updated property in the credentials - // available to the authenticate method - Object.assign(credentialsDecrypted, data); - } - - requestOptions = (await additionalData.credentialsHelper.authenticate( - credentialsDecrypted, - credentialsType, - requestOptions as IHttpRequestOptions, - workflow, - node, - )) as IRequestOptions; - return await proxyRequestToAxios(workflow, additionalData, node, requestOptions); - } catch (error) { - try { - if (credentialsDecrypted !== undefined) { - // try to refresh the credentials - const data = await additionalData.credentialsHelper.preAuthentication( - { helpers: this.helpers }, - credentialsDecrypted, - credentialsType, - node, - true, - ); - - if (data) { - // make the updated property in the credentials - // available to the authenticate method - Object.assign(credentialsDecrypted, data); - requestOptions = (await additionalData.credentialsHelper.authenticate( - credentialsDecrypted, - credentialsType, - requestOptions as IHttpRequestOptions, - workflow, - node, - )) as IRequestOptions; - // retry the request - return await proxyRequestToAxios(workflow, additionalData, node, requestOptions); - } - } - throw error; - } catch (error) { - if (error instanceof ExecutionBaseError) throw error; - - throw new NodeApiError(this.getNode(), error); - } - } -} - -/** - * Returns the webhook URL of the webhook with the given name - */ -export function getNodeWebhookUrl( - name: WebhookType, - workflow: Workflow, - node: INode, - additionalData: IWorkflowExecuteAdditionalData, - mode: WorkflowExecuteMode, - additionalKeys: IWorkflowDataProxyAdditionalKeys, - isTest?: boolean, -): string | undefined { - let baseUrl = additionalData.webhookBaseUrl; - if (isTest === true) { - baseUrl = additionalData.webhookTestBaseUrl; - } - - // eslint-disable-next-line @typescript-eslint/no-use-before-define - const webhookDescription = getWebhookDescription(name, workflow, node); - if (webhookDescription === undefined) { - return undefined; - } - - const path = workflow.expression.getSimpleParameterValue( - node, - webhookDescription.path, - mode, - additionalKeys, - ); - if (path === undefined) { - return undefined; - } - - const isFullPath: boolean = workflow.expression.getSimpleParameterValue( - node, - webhookDescription.isFullPath, - mode, - additionalKeys, - undefined, - false, - ) as boolean; - return NodeHelpers.getNodeWebhookUrl(baseUrl, workflow.id, node, path.toString(), isFullPath); -} - -/** - * Returns the full webhook description of the webhook with the given name - */ -export function getWebhookDescription( - name: WebhookType, - workflow: Workflow, - node: INode, -): IWebhookDescription | undefined { - const nodeType = workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion); - - if (nodeType.description.webhooks === undefined) { - // Node does not have any webhooks so return - return undefined; - } - - for (const webhookDescription of nodeType.description.webhooks) { - if (webhookDescription.name === name) { - return webhookDescription; - } - } - - return undefined; -} - -export const getRequestHelperFunctions = ( - workflow: Workflow, - node: INode, - additionalData: IWorkflowExecuteAdditionalData, - runExecutionData: IRunExecutionData | null = null, - connectionInputData: INodeExecutionData[] = [], -): RequestHelperFunctions => { - const getResolvedValue = ( - parameterValue: NodeParameterValueType, - itemIndex: number, - runIndex: number, - executeData: IExecuteData, - additionalKeys?: IWorkflowDataProxyAdditionalKeys, - returnObjectAsString = false, - ): NodeParameterValueType => { - const mode: WorkflowExecuteMode = 'internal'; - - if ( - typeof parameterValue === 'object' || - (typeof parameterValue === 'string' && parameterValue.charAt(0) === '=') - ) { - return workflow.expression.getParameterValue( - parameterValue, - runExecutionData, - runIndex, - itemIndex, - node.name, - connectionInputData, - mode, - additionalKeys ?? {}, - executeData, - returnObjectAsString, - ); - } - - return parameterValue; - }; - - return { - httpRequest, - // eslint-disable-next-line complexity - async requestWithAuthenticationPaginated( - this: IExecuteFunctions, - requestOptions: IRequestOptions, - itemIndex: number, - paginationOptions: PaginationOptions, - credentialsType?: string, - additionalCredentialOptions?: IAdditionalCredentialOptions, - ): Promise { - const responseData = []; - if (!requestOptions.qs) { - requestOptions.qs = {}; - } - requestOptions.resolveWithFullResponse = true; - requestOptions.simple = false; - - let tempResponseData: IN8nHttpFullResponse; - let makeAdditionalRequest: boolean; - let paginateRequestData: PaginationOptions['request']; - - const runIndex = 0; - - const additionalKeys: IWorkflowDataProxyAdditionalKeys = { - $request: requestOptions, - $response: {} as IN8nHttpFullResponse, - $version: node.typeVersion, - $pageCount: 0, - }; - - const executeData: IExecuteData = { - data: {}, - node, - source: null, - }; - - const hashData = { - identicalCount: 0, - previousLength: 0, - previousHash: '', - }; - do { - paginateRequestData = getResolvedValue( - paginationOptions.request as unknown as NodeParameterValueType, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as object as PaginationOptions['request']; - - const tempRequestOptions = applyPaginationRequestData(requestOptions, paginateRequestData); - - if (!validateUrl(tempRequestOptions.uri as string)) { - throw new NodeOperationError(node, `'${paginateRequestData.url}' is not a valid URL.`, { - itemIndex, - runIndex, - type: 'invalid_url', - }); - } - - if (credentialsType) { - tempResponseData = await this.helpers.requestWithAuthentication.call( - this, - credentialsType, - tempRequestOptions, - additionalCredentialOptions, - ); - } else { - tempResponseData = await this.helpers.request(tempRequestOptions); - } - - const newResponse: IN8nHttpFullResponse = Object.assign( - { - body: {}, - headers: {}, - statusCode: 0, - }, - pick(tempResponseData, ['body', 'headers', 'statusCode']), - ); - - let contentBody: Exclude; - - if (newResponse.body instanceof Readable && paginationOptions.binaryResult !== true) { - // Keep the original string version that we can use it to hash if needed - contentBody = await binaryToString(newResponse.body as Buffer | Readable); - - const responseContentType = newResponse.headers['content-type']?.toString() ?? ''; - if (responseContentType.includes('application/json')) { - newResponse.body = jsonParse(contentBody, { fallbackValue: {} }); - } else { - newResponse.body = contentBody; - } - tempResponseData.__bodyResolved = true; - tempResponseData.body = newResponse.body; - } else { - contentBody = newResponse.body; - } - - if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) { - // If the data is not binary (and so not a stream), or an etag is present, - // we check via etag or hash if identical data is received - - let contentLength = 0; - if ('content-length' in tempResponseData.headers) { - contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0; - } - - if (hashData.previousLength === contentLength) { - let hash: string; - if (tempResponseData.headers.etag) { - // If an etag is provided, we use it as "hash" - hash = tempResponseData.headers.etag as string; - } else { - // If there is no etag, we calculate a hash from the data in the body - if (typeof contentBody !== 'string') { - contentBody = JSON.stringify(contentBody); - } - hash = crypto.createHash('md5').update(contentBody).digest('base64'); - } - - if (hashData.previousHash === hash) { - hashData.identicalCount += 1; - if (hashData.identicalCount > 2) { - // Length was identical 5x and hash 3x - throw new NodeOperationError( - node, - 'The returned response was identical 5x, so requests got stopped', - { - itemIndex, - description: - 'Check if "Pagination Completed When" has been configured correctly.', - }, - ); - } - } else { - hashData.identicalCount = 0; - } - hashData.previousHash = hash; - } else { - hashData.identicalCount = 0; - } - hashData.previousLength = contentLength; - } - - responseData.push(tempResponseData); - - additionalKeys.$response = newResponse; - additionalKeys.$pageCount = (additionalKeys.$pageCount ?? 0) + 1; - - const maxRequests = getResolvedValue( - paginationOptions.maxRequests, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as number; - - if (maxRequests && additionalKeys.$pageCount >= maxRequests) { - break; - } - - makeAdditionalRequest = getResolvedValue( - paginationOptions.continue, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as boolean; - - if (makeAdditionalRequest) { - if (paginationOptions.requestInterval) { - const requestInterval = getResolvedValue( - paginationOptions.requestInterval, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as number; - - await sleep(requestInterval); - } - if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) { - // We have it configured to let all requests pass no matter the response code - // via "requestOptions.simple = false" to not by default fail if it is for example - // configured to stop on 404 response codes. For that reason we have to throw here - // now an error manually if the response code is not a success one. - let data = tempResponseData.body; - if (data instanceof Readable && paginationOptions.binaryResult !== true) { - data = await binaryToString(data as Buffer | Readable); - } else if (typeof data === 'object') { - data = JSON.stringify(data); - } - - throw Object.assign( - new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`), - { - statusCode: tempResponseData.statusCode, - error: data, - isAxiosError: true, - response: { - headers: tempResponseData.headers, - status: tempResponseData.statusCode, - statusText: tempResponseData.statusMessage, - }, - }, - ); - } - } - } while (makeAdditionalRequest); - - return responseData; - }, - async httpRequestWithAuthentication( - this, - credentialsType, - requestOptions, - additionalCredentialOptions, - ): Promise { - return await httpRequestWithAuthentication.call( - this, - credentialsType, - requestOptions, - workflow, - node, - additionalData, - additionalCredentialOptions, - ); - }, - - request: async (uriOrObject, options) => - await proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options), - - async requestWithAuthentication( - this, - credentialsType, - requestOptions, - additionalCredentialOptions, - itemIndex, - ): Promise { - return await requestWithAuthentication.call( - this, - credentialsType, - requestOptions, - workflow, - node, - additionalData, - additionalCredentialOptions, - itemIndex, - ); - }, - - async requestOAuth1( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IRequestOptions, - ): Promise { - return await requestOAuth1.call(this, credentialsType, requestOptions); - }, - - async requestOAuth2( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: IRequestOptions, - oAuth2Options?: IOAuth2Options, - ): Promise { - return await requestOAuth2.call( - this, - credentialsType, - requestOptions, - node, - additionalData, - oAuth2Options, - ); - }, - }; -}; - -export const getSSHTunnelFunctions = (): SSHTunnelFunctions => ({ - getSSHClient: async (credentials) => - await Container.get(SSHClientsManager).getClient(credentials), -}); - -export const getSchedulingFunctions = (workflow: Workflow): SchedulingFunctions => { - const scheduledTaskManager = Container.get(ScheduledTaskManager); - return { - registerCron: (cronExpression, onTick) => - scheduledTaskManager.registerCron(workflow, cronExpression, onTick), - }; -}; - -/** - * Returns a copy of the items which only contains the json data and - * of that only the defined properties - */ -export function copyInputItems(items: INodeExecutionData[], properties: string[]): IDataObject[] { - return items.map((item) => { - const newItem: IDataObject = {}; - for (const property of properties) { - if (item.json[property] === undefined) { - newItem[property] = null; - } else { - newItem[property] = deepCopy(item.json[property]); - } - } - return newItem; - }); -} +import { PollContext, TriggerContext } from './execution-engine/node-execution-context'; /** * Returns the execute functions the poll nodes have access to. diff --git a/packages/nodes-base/nodes/Google/Drive/test/v2/node/helpers.ts b/packages/nodes-base/nodes/Google/Drive/test/v2/node/helpers.ts index 780de933af..2dd1912a61 100644 --- a/packages/nodes-base/nodes/Google/Drive/test/v2/node/helpers.ts +++ b/packages/nodes-base/nodes/Google/Drive/test/v2/node/helpers.ts @@ -1,5 +1,5 @@ import { get } from 'lodash'; -import { constructExecutionMetaData, returnJsonArray } from 'n8n-core'; +import { constructExecutionMetaData } from 'n8n-core'; import type { IDataObject, IExecuteFunctions, IGetNodeParameterOptions, INode } from 'n8n-workflow'; import { Readable } from 'stream'; @@ -32,7 +32,7 @@ export const createMockExecuteFunction = ( }, helpers: { constructExecutionMetaData, - returnJsonArray, + returnJsonArray: () => [], prepareBinaryData: () => {}, httpRequest: () => {}, },