From cc2bd2e19c8b75320b236de215d389220fbe24ae Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Wed, 1 Nov 2023 14:24:43 +0100 Subject: [PATCH] feat(HTTP Request Node): Add pagination support (#5993) Is still WIP and does not implement the correct UI yet. Github issue / Community forum post (link here to close automatically): https://community.n8n.io/t/pagination-included-into-http-node/15080 https://community.n8n.io/t/how-to-paginate-through-data-in-http-requests/28103 --- packages/core/src/Constants.ts | 1 + packages/core/src/NodeExecuteFunctions.ts | 351 +++- .../editor-ui/src/mixins/workflowHelpers.ts | 13 +- .../HttpRequest/V3/HttpRequestV3.node.ts | 681 ++++++-- .../HttpRequest/test/node/HttpRequest.test.ts | 43 + .../test/node/workflow.pagination.json | 1537 +++++++++++++++++ packages/workflow/src/Interfaces.ts | 20 +- 7 files changed, 2408 insertions(+), 238 deletions(-) create mode 100644 packages/nodes-base/nodes/HttpRequest/test/node/workflow.pagination.json diff --git a/packages/core/src/Constants.ts b/packages/core/src/Constants.ts index 4ce7b66341..dd6277b69b 100644 --- a/packages/core/src/Constants.ts +++ b/packages/core/src/Constants.ts @@ -1,6 +1,7 @@ export const CUSTOM_EXTENSION_ENV = 'N8N_CUSTOM_EXTENSIONS'; export const PLACEHOLDER_EMPTY_EXECUTION_ID = '__UNKNOWN__'; export const PLACEHOLDER_EMPTY_WORKFLOW_ID = '__EMPTY__'; +export const HTTP_REQUEST_NODE_TYPE = 'n8n-nodes-base.httpRequest'; export const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 95b92bb3a1..eaab998397 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -37,6 +37,7 @@ import { extension, lookup } from 'mime-types'; import type { BinaryHelperFunctions, ConnectionTypes, + ContextType, ExecutionError, FieldType, FileSystemHelperFunctions, @@ -88,6 +89,7 @@ import type { NodeExecutionWithMetadata, NodeHelperFunctions, NodeParameterValueType, + PaginationOptions, RequestHelperFunctions, Workflow, WorkflowActivateMode, @@ -110,13 +112,14 @@ import { isResourceMapperValue, validateFieldType, ExecutionBaseError, + jsonParse, } from 'n8n-workflow'; import type { Token } from 'oauth-1.0a'; import clientOAuth1 from 'oauth-1.0a'; import path from 'path'; import { stringify } from 'qs'; -import type { OptionsWithUri, OptionsWithUrl } from 'request'; -import type { RequestPromiseOptions } from 'request-promise-native'; +import type { OptionsWithUrl } from 'request'; +import type { OptionsWithUri, RequestPromiseOptions } from 'request-promise-native'; import { Readable } from 'stream'; import url, { URL, URLSearchParams } from 'url'; @@ -126,6 +129,7 @@ import { BLOCK_FILE_ACCESS_TO_N8N_FILES, CONFIG_FILES, CUSTOM_EXTENSION_ENV, + HTTP_REQUEST_NODE_TYPE, PLACEHOLDER_EMPTY_EXECUTION_ID, RESTRICT_FILE_ACCESS_TO, UM_EMAIL_TEMPLATES_INVITE, @@ -143,6 +147,7 @@ import { import { getSecretsProxy } from './Secrets'; import Container from 'typedi'; import type { BinaryData } from './BinaryData/types'; +import merge from 'lodash/merge'; import { InstanceSettings } from './InstanceSettings'; axios.defaults.timeout = 300000; @@ -1866,7 +1871,7 @@ export async function getCredentials( // Hardcode for now for security reasons that only a single node can access // all credentials - const fullAccess = ['n8n-nodes-base.httpRequest'].includes(node.type); + const fullAccess = [HTTP_REQUEST_NODE_TYPE].includes(node.type); let nodeCredentialDescription: INodeCredentialDescription | undefined; if (!fullAccess) { @@ -2239,6 +2244,7 @@ export function getNodeParameter( } let returnData; + try { returnData = workflow.expression.getParameterValue( value, @@ -2506,70 +2512,303 @@ const getRequestHelperFunctions = ( workflow: Workflow, node: INode, additionalData: IWorkflowExecuteAdditionalData, -): RequestHelperFunctions => ({ - httpRequest, +): RequestHelperFunctions => { + const getResolvedValue = ( + parameterValue: NodeParameterValueType, + itemIndex: number, + runIndex: number, + executeData: IExecuteData, + additionalKeys?: IWorkflowDataProxyAdditionalKeys, + returnObjectAsString = false, + ): NodeParameterValueType => { + const runExecutionData: IRunExecutionData | null = null; + const connectionInputData: INodeExecutionData[] = []; + const mode: WorkflowExecuteMode = 'internal'; - async httpRequestWithAuthentication( - this, - credentialsType, - requestOptions, - additionalCredentialOptions, - ): Promise { - return httpRequestWithAuthentication.call( + if ( + typeof parameterValue === 'object' || + (typeof parameterValue === 'string' && parameterValue.charAt(0) === '=') + ) { + return workflow.expression.getParameterValue( + parameterValue, + runExecutionData, + runIndex, + itemIndex, + node.name, + connectionInputData, + mode, + additionalKeys ?? {}, + executeData, + returnObjectAsString, + ); + } + + return parameterValue; + }; + + return { + httpRequest, + async requestWithAuthenticationPaginated( + this: IExecuteFunctions, + requestOptions: OptionsWithUri, + itemIndex: number, + paginationOptions: PaginationOptions, + credentialsType?: string, + additionalCredentialOptions?: IAdditionalCredentialOptions, + ): Promise { + const responseData = []; + if (!requestOptions.qs) { + requestOptions.qs = {}; + } + requestOptions.resolveWithFullResponse = true; + requestOptions.simple = false; + + let tempResponseData: IN8nHttpFullResponse; + let makeAdditionalRequest: boolean; + let paginateRequestData: IHttpRequestOptions; + + const runIndex = 0; + + const additionalKeys = { + $request: requestOptions, + $response: {} as IN8nHttpFullResponse, + $version: node.typeVersion, + $pageCount: 0, + }; + + const executeData: IExecuteData = { + data: {}, + node, + source: null, + }; + + const hashData = { + identicalCount: 0, + previousLength: 0, + previousHash: '', + }; + do { + paginateRequestData = getResolvedValue( + paginationOptions.request as unknown as NodeParameterValueType, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as object as IHttpRequestOptions; + + const tempRequestOptions = merge(requestOptions, paginateRequestData); + + if (credentialsType) { + tempResponseData = await this.helpers.requestWithAuthentication.call( + this, + credentialsType, + tempRequestOptions, + additionalCredentialOptions, + ); + } else { + tempResponseData = await this.helpers.request(tempRequestOptions); + } + + const newResponse: IN8nHttpFullResponse = Object.assign( + { + body: {}, + headers: {}, + statusCode: 0, + }, + pick(tempResponseData, ['body', 'headers', 'statusCode']), + ); + + let contentBody: Exclude; + + if ( + newResponse.body?.constructor.name === 'IncomingMessage' && + paginationOptions.binaryResult !== true + ) { + const data = await this.helpers + .binaryToBuffer(newResponse.body as Buffer | Readable) + .then((body) => body.toString()); + // Keep the original string version that we can use it to hash if needed + contentBody = data; + + const responseContentType = newResponse.headers['content-type']?.toString() ?? ''; + if (responseContentType.includes('application/json')) { + newResponse.body = jsonParse(data, { fallbackValue: {} }); + } else { + newResponse.body = data; + } + tempResponseData.__bodyResolved = true; + tempResponseData.body = newResponse.body; + } else { + contentBody = newResponse.body; + } + + if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) { + // If the data is not binary (and so not a stream), or an etag is present, + // we check via etag or hash if identical data is received + + let contentLength = 0; + if ('content-length' in tempResponseData.headers) { + contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0; + } + + if (hashData.previousLength === contentLength) { + let hash: string; + if (tempResponseData.headers.etag) { + // If an etag is provided, we use it as "hash" + hash = tempResponseData.headers.etag as string; + } else { + // If there is no etag, we calculate a hash from the data in the body + if (typeof contentBody !== 'string') { + contentBody = JSON.stringify(contentBody); + } + hash = crypto.createHash('md5').update(contentBody).digest('base64'); + } + + if (hashData.previousHash === hash) { + hashData.identicalCount += 1; + if (hashData.identicalCount > 2) { + // Length was identical 5x and hash 3x + throw new NodeOperationError( + node, + 'The returned response was identical 5x, so requests got stopped', + { + itemIndex, + description: + 'Check if "Pagination Completed When" has been configured correctly.', + }, + ); + } + } else { + hashData.identicalCount = 0; + } + hashData.previousHash = hash; + } else { + hashData.identicalCount = 0; + } + hashData.previousLength = contentLength; + } + + responseData.push(tempResponseData); + + additionalKeys.$response = newResponse; + additionalKeys.$pageCount = additionalKeys.$pageCount + 1; + + if ( + paginationOptions.maxRequests && + additionalKeys.$pageCount >= paginationOptions.maxRequests + ) { + break; + } + + makeAdditionalRequest = getResolvedValue( + paginationOptions.continue, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as boolean; + + if (makeAdditionalRequest) { + if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) { + // We have it configured to let all requests pass no matter the response code + // via "requestOptions.simple = false" to not by default fail if it is for example + // configured to stop on 404 response codes. For that reason we have to throw here + // now an error manually if the response code is not a success one. + let data = tempResponseData.body; + if ( + data?.constructor.name === 'IncomingMessage' && + paginationOptions.binaryResult !== true + ) { + data = await this.helpers + .binaryToBuffer(tempResponseData.body as Buffer | Readable) + .then((body) => body.toString()); + } else if (typeof data === 'object') { + data = JSON.stringify(data); + } + + throw Object.assign( + new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`), + { + statusCode: tempResponseData.statusCode, + error: data, + isAxiosError: true, + response: { + headers: tempResponseData.headers, + status: tempResponseData.statusCode, + statusText: tempResponseData.statusMessage, + }, + }, + ); + } + } + } while (makeAdditionalRequest); + + return responseData; + }, + async httpRequestWithAuthentication( this, credentialsType, requestOptions, - workflow, - node, - additionalData, additionalCredentialOptions, - ); - }, + ): Promise { + return httpRequestWithAuthentication.call( + this, + credentialsType, + requestOptions, + workflow, + node, + additionalData, + additionalCredentialOptions, + ); + }, - request: async (uriOrObject, options) => - proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options), + request: async (uriOrObject, options) => + proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options), - async requestWithAuthentication( - this, - credentialsType, - requestOptions, - additionalCredentialOptions, - ): Promise { - return requestWithAuthentication.call( + async requestWithAuthentication( this, credentialsType, requestOptions, - workflow, - node, - additionalData, additionalCredentialOptions, - ); - }, + ): Promise { + return requestWithAuthentication.call( + this, + credentialsType, + requestOptions, + workflow, + node, + additionalData, + additionalCredentialOptions, + ); + }, - async requestOAuth1( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: OptionsWithUrl | RequestPromiseOptions, - ): Promise { - return requestOAuth1.call(this, credentialsType, requestOptions); - }, + async requestOAuth1( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: OptionsWithUrl | RequestPromiseOptions, + ): Promise { + return requestOAuth1.call(this, credentialsType, requestOptions); + }, - async requestOAuth2( - this: IAllExecuteFunctions, - credentialsType: string, - requestOptions: OptionsWithUri | RequestPromiseOptions, - oAuth2Options?: IOAuth2Options, - ): Promise { - return requestOAuth2.call( - this, - credentialsType, - requestOptions, - node, - additionalData, - oAuth2Options, - ); - }, -}); + async requestOAuth2( + this: IAllExecuteFunctions, + credentialsType: string, + requestOptions: OptionsWithUri | RequestPromiseOptions, + oAuth2Options?: IOAuth2Options, + ): Promise { + return requestOAuth2.call( + this, + credentialsType, + requestOptions, + node, + additionalData, + oAuth2Options, + ); + }, + }; +}; const getAllowedPaths = () => { const restrictFileAccessTo = process.env[RESTRICT_FILE_ACCESS_TO]; @@ -2899,7 +3138,7 @@ export function getExecuteFunctions( ), ); }, - getContext(type: string): IContextObject { + getContext(type: ContextType): IContextObject { return NodeHelpers.getContext(runExecutionData, type, node); }, async getInputConnectionData( @@ -3293,7 +3532,7 @@ export function getExecuteSingleFunctions( executeData, ); }, - getContext(type: string): IContextObject { + getContext(type: ContextType): IContextObject { return NodeHelpers.getContext(runExecutionData, type, node); }, getCredentials: async (type) => diff --git a/packages/editor-ui/src/mixins/workflowHelpers.ts b/packages/editor-ui/src/mixins/workflowHelpers.ts index b695a06e0c..d10b1bb7c1 100644 --- a/packages/editor-ui/src/mixins/workflowHelpers.ts +++ b/packages/editor-ui/src/mixins/workflowHelpers.ts @@ -1,5 +1,6 @@ import { EnterpriseEditionFeature, + HTTP_REQUEST_NODE_TYPE, MODAL_CONFIRM, PLACEHOLDER_EMPTY_WORKFLOW_ID, PLACEHOLDER_FILLED_AT_EXECUTION_TIME, @@ -49,7 +50,7 @@ import { externalHooks } from '@/mixins/externalHooks'; import { genericHelpers } from '@/mixins/genericHelpers'; import { nodeHelpers } from '@/mixins/nodeHelpers'; -import { isEqual } from 'lodash-es'; +import { get, isEqual } from 'lodash-es'; import type { IPermissions } from '@/permissions'; import { getWorkflowPermissions } from '@/permissions'; @@ -194,6 +195,16 @@ export function resolveParameter( ...opts.additionalKeys, }; + if (activeNode?.type === HTTP_REQUEST_NODE_TYPE) { + // Add $response for HTTP Request-Nodes as it is used + // in pagination expressions + additionalKeys.$response = get( + executionData, + `data.executionData.contextData['node:${activeNode!.name}'].response`, + {}, + ); + } + let runIndexCurrent = opts?.targetItem?.runIndex ?? 0; if ( opts?.targetItem === undefined && diff --git a/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts b/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts index cfd7a2a500..fc6a473fc8 100644 --- a/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts +++ b/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts @@ -9,6 +9,7 @@ import type { INodeTypeBaseDescription, INodeTypeDescription, IRequestOptionsSimplified, + PaginationOptions, JsonObject, } from 'n8n-workflow'; @@ -927,6 +928,228 @@ export class HttpRequestV3 implements INodeType { }, ], }, + { + displayName: 'Pagination', + name: 'pagination', + placeholder: 'Add pagination', + type: 'fixedCollection', + typeOptions: { + multipleValues: false, + }, + default: { + pagination: {}, + }, + options: [ + { + displayName: 'Pagination', + name: 'pagination', + values: [ + { + displayName: 'Pagination Mode', + name: 'paginationMode', + type: 'options', + typeOptions: { + noDataExpression: true, + }, + options: [ + { + name: 'Off', + value: 'off', + }, + { + name: 'Update a Parameter in Each Request', + value: 'updateAParameterInEachRequest', + }, + { + name: 'Response Contains Next URL', + value: 'responseContainsNextURL', + }, + ], + default: 'updateAParameterInEachRequest', + description: 'If pagination should be used', + }, + { + displayName: + 'Use the $response variables to access the data of the previous response. More info', + name: 'webhookNotice', + displayOptions: { + hide: { + paginationMode: ['off'], + }, + }, + type: 'notice', + default: '', + }, + { + displayName: 'Next URL', + name: 'nextURL', + type: 'string', + displayOptions: { + show: { + paginationMode: ['responseContainsNextURL'], + }, + }, + default: '', + description: + 'Should evaluate to true when pagination is complete. More info.', + }, + { + displayName: 'Parameters', + name: 'parameters', + type: 'fixedCollection', + displayOptions: { + show: { + paginationMode: ['updateAParameterInEachRequest'], + }, + }, + typeOptions: { + multipleValues: true, + noExpression: true, + }, + placeholder: 'Add Parameter', + default: { + parameters: [ + { + type: 'qs', + name: '', + value: '', + }, + ], + }, + options: [ + { + name: 'parameters', + displayName: 'Parameter', + values: [ + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Body', + value: 'body', + }, + { + name: 'Header', + value: 'headers', + }, + { + name: 'Query', + value: 'qs', + }, + ], + default: 'qs', + description: 'Where the parameter should be set', + }, + { + displayName: 'Name', + name: 'name', + type: 'string', + default: '', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + { + displayName: 'Pagination Complete When', + name: 'paginationCompleteWhen', + type: 'options', + typeOptions: { + noDataExpression: true, + }, + displayOptions: { + hide: { + paginationMode: ['off'], + }, + }, + options: [ + { + name: 'Response Is Empty', + value: 'responseIsEmpty', + }, + { + name: 'Receive Specific Status Code(s)', + value: 'receiveSpecificStatusCodes', + }, + { + name: 'Other', + value: 'other', + }, + ], + default: 'responseIsEmpty', + description: 'When should no further requests be made?', + }, + { + displayName: 'Status Code(s) when Complete', + name: 'statusCodesWhenComplete', + type: 'string', + typeOptions: { + noDataExpression: true, + }, + displayOptions: { + show: { + paginationCompleteWhen: ['receiveSpecificStatusCodes'], + }, + }, + default: '', + description: 'Accepts comma-separated values', + }, + { + displayName: 'Complete Expression', + name: 'completeExpression', + type: 'string', + displayOptions: { + show: { + paginationCompleteWhen: ['other'], + }, + }, + default: '', + description: + 'Should evaluate to true when pagination is complete. More info.', + }, + { + displayName: 'Limit Pages Fetched', + name: 'limitPagesFetched', + type: 'boolean', + typeOptions: { + noDataExpression: true, + }, + displayOptions: { + hide: { + paginationMode: ['off'], + }, + }, + default: false, + noDataExpression: true, + description: 'Whether the number of requests should be limited', + }, + { + displayName: 'Max Pages', + name: 'maxRequests', + type: 'number', + typeOptions: { + noDataExpression: true, + }, + displayOptions: { + show: { + limitPagesFetched: [true], + }, + }, + default: 100, + description: 'Maximum amount of request to be make', + }, + ], + }, + ], + }, { displayName: 'Proxy', name: 'proxy', @@ -1033,6 +1256,26 @@ export class HttpRequestV3 implements INodeType { let autoDetectResponseFormat = false; + // Can not be defined on a per item level + const pagination = this.getNodeParameter('options.pagination.pagination', 0, null, { + rawExpressions: true, + }) as { + paginationMode: 'off' | 'updateAParameterInEachRequest' | 'responseContainsNextURL'; + nextURL?: string; + parameters: { + parameters: Array<{ + type: 'body' | 'headers' | 'qs'; + name: string; + value: string; + }>; + }; + paginationCompleteWhen: 'responseIsEmpty' | 'receiveSpecificStatusCodes' | 'other'; + statusCodesWhenComplete: string; + completeExpression: string; + limitPagesFetched: boolean; + maxRequests: number; + }; + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { const requestMethod = this.getNodeParameter('method', itemIndex) as string; @@ -1117,15 +1360,9 @@ export class HttpRequestV3 implements INodeType { gzip: true, rejectUnauthorized: !allowUnauthorizedCerts || false, followRedirect: false, + resolveWithFullResponse: true, }; - // When response format is set to auto-detect, - // we need to access to response header content-type - // and the only way is using "resolveWithFullResponse" - if (autoDetectResponseFormat || fullResponse) { - requestOptions.resolveWithFullResponse = true; - } - if (requestOptions.method !== 'GET' && nodeVersion >= 4.1) { requestOptions = { ...requestOptions, followAllRedirects: false }; } @@ -1321,7 +1558,7 @@ export class HttpRequestV3 implements INodeType { requestOptions.json = true; } - // // Add Content Type if any are set + // Add Content Type if any are set if (bodyContentType === 'raw') { if (requestOptions.headers === undefined) { requestOptions.headers = {}; @@ -1392,7 +1629,64 @@ export class HttpRequestV3 implements INodeType { try { this.sendMessageToUI(sanitizeUiMessage(requestOptions, authDataKeys)); } catch (e) {} - if (authentication === 'genericCredentialType' || authentication === 'none') { + + if (pagination && pagination.paginationMode !== 'off') { + let continueExpression = '={{false}}'; + if (pagination.paginationCompleteWhen === 'receiveSpecificStatusCodes') { + // Split out comma separated list of status codes into array + const statusCodesWhenCompleted = pagination.statusCodesWhenComplete + .split(',') + .map((item) => parseInt(item.trim())); + + continueExpression = `={{ !${JSON.stringify( + statusCodesWhenCompleted, + )}.includes($response.statusCode) }}`; + } else if (pagination.paginationCompleteWhen === 'responseIsEmpty') { + continueExpression = + '={{ Array.isArray($response.body) ? $response.body.length : !!$response.body }}'; + } else { + // Other + if (!pagination.completeExpression.length || pagination.completeExpression[0] !== '=') { + throw new NodeOperationError(this.getNode(), 'Invalid or empty Complete Expression'); + } + continueExpression = `={{ !(${pagination.completeExpression.trim().slice(3, -2)}) }}`; + } + + const paginationData: PaginationOptions = { + continue: continueExpression, + request: {}, + }; + + if (pagination.paginationMode === 'updateAParameterInEachRequest') { + // Iterate over all parameters and add them to the request + paginationData.request = {}; + pagination.parameters.parameters.forEach((parameter) => { + if (!paginationData.request[parameter.type]) { + paginationData.request[parameter.type] = {}; + } + paginationData.request[parameter.type]![parameter.name] = parameter.value; + }); + } else if (pagination.paginationMode === 'responseContainsNextURL') { + paginationData.request.url = pagination.nextURL; + } + + if (pagination.limitPagesFetched) { + paginationData.maxRequests = pagination.maxRequests; + } + + if (responseFormat === 'file') { + paginationData.binaryResult = true; + } + + const requestPromise = this.helpers.requestWithAuthenticationPaginated.call( + this, + requestOptions, + itemIndex, + paginationData, + nodeCredentialType, + ); + requestPromises.push(requestPromise); + } else if (authentication === 'genericCredentialType' || authentication === 'none') { if (oAuth1Api) { const requestOAuth1 = this.helpers.requestOAuth1.call(this, 'oAuth1Api', requestOptions); requestOAuth1.catch(() => {}); @@ -1426,25 +1720,25 @@ export class HttpRequestV3 implements INodeType { } const promisesResponses = await Promise.allSettled(requestPromises); - let response: any; + let responseData: any; for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { - response = promisesResponses.shift(); - if (response!.status !== 'fulfilled') { - if (response.reason.statusCode === 429) { - response.reason.message = + responseData = promisesResponses.shift(); + if (responseData!.status !== 'fulfilled') { + if (responseData.reason.statusCode === 429) { + responseData.reason.message = "Try spacing your requests out using the batching settings under 'Options'"; } if (!this.continueOnFail()) { - if (autoDetectResponseFormat && response.reason.error instanceof Buffer) { - response.reason.error = Buffer.from(response.reason.error as Buffer).toString(); + if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) { + responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString(); } - throw new NodeApiError(this.getNode(), response as JsonObject, { itemIndex }); + throw new NodeApiError(this.getNode(), responseData as JsonObject, { itemIndex }); } else { - removeCircularRefs(response.reason as JsonObject); + removeCircularRefs(responseData.reason as JsonObject); // Return the actual reason as error returnItems.push({ json: { - error: response.reason, + error: responseData.reason, }, pairedItem: { item: itemIndex, @@ -1454,7 +1748,12 @@ export class HttpRequestV3 implements INodeType { } } - response = response.value; + let responses: any[]; + if (Array.isArray(responseData.value)) { + responses = responseData.value; + } else { + responses = [responseData.value]; + } let responseFormat = this.getNodeParameter( 'options.response.response.responseFormat', @@ -1468,172 +1767,194 @@ export class HttpRequestV3 implements INodeType { false, ) as boolean; - if (autoDetectResponseFormat) { - const responseContentType = response.headers['content-type'] ?? ''; - if (responseContentType.includes('application/json')) { - responseFormat = 'json'; - const neverError = this.getNodeParameter( - 'options.response.response.neverError', - 0, - false, - ) as boolean; - - const data = await this.helpers - .binaryToBuffer(response.body as Buffer | Readable) - .then((body) => body.toString()); - response.body = jsonParse(data, { - ...(neverError - ? { fallbackValue: {} } - : { errorMessage: 'Invalid JSON in response body' }), - }); - } else if (binaryContentTypes.some((e) => responseContentType.includes(e))) { - responseFormat = 'file'; - } else { - responseFormat = 'text'; - const data = await this.helpers - .binaryToBuffer(response.body as Buffer | Readable) - .then((body) => body.toString()); - response.body = !data ? undefined : data; - } - } - - if (autoDetectResponseFormat && !fullResponse) { - delete response.headers; - delete response.statusCode; - delete response.statusMessage; - response = response.body; - requestOptions.resolveWithFullResponse = false; - } - - if (responseFormat === 'file') { - const outputPropertyName = this.getNodeParameter( - 'options.response.response.outputPropertyName', - 0, - 'data', - ) as string; - - const newItem: INodeExecutionData = { - json: {}, - binary: {}, - pairedItem: { - item: itemIndex, - }, - }; - - if (items[itemIndex].binary !== undefined) { - // Create a shallow copy of the binary data so that the old - // data references which do not get changed still stay behind - // but the incoming data does not get changed. - Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary); - } - - let binaryData: Buffer | Readable; - if (fullResponse) { - const returnItem: IDataObject = {}; - for (const property of fullResponseProperties) { - if (property === 'body') { - continue; - } - returnItem[property] = response![property]; - } - - newItem.json = returnItem; - binaryData = response!.body; - } else { - newItem.json = items[itemIndex].json; - binaryData = response; - } - newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(binaryData); - - returnItems.push(newItem); - } else if (responseFormat === 'text') { - const outputPropertyName = this.getNodeParameter( - 'options.response.response.outputPropertyName', - 0, - 'data', - ) as string; - if (fullResponse) { - const returnItem: IDataObject = {}; - for (const property of fullResponseProperties) { - if (property === 'body') { - returnItem[outputPropertyName] = toText(response![property]); - continue; - } - - returnItem[property] = response![property]; - } - returnItems.push({ - json: returnItem, - pairedItem: { - item: itemIndex, - }, - }); - } else { - returnItems.push({ - json: { - [outputPropertyName]: toText(response), - }, - pairedItem: { - item: itemIndex, - }, - }); - } - } else { - // responseFormat: 'json' - if (requestOptions.resolveWithFullResponse === true) { - const returnItem: IDataObject = {}; - for (const property of fullResponseProperties) { - returnItem[property] = response![property]; - } - - if (responseFormat === 'json' && typeof returnItem.body === 'string') { - try { - returnItem.body = JSON.parse(returnItem.body); - } catch (error) { - throw new NodeOperationError( - this.getNode(), - 'Response body is not valid JSON. Change "Response Format" to "Text"', - { itemIndex }, - ); - } - } - - returnItems.push({ - json: returnItem, - pairedItem: { - item: itemIndex, - }, - }); - } else { - if (responseFormat === 'json' && typeof response === 'string') { - try { - response = JSON.parse(response); - } catch (error) { - throw new NodeOperationError( - this.getNode(), - 'Response body is not valid JSON. Change "Response Format" to "Text"', - { itemIndex }, - ); - } - } - - if (Array.isArray(response)) { - // eslint-disable-next-line @typescript-eslint/no-loop-func - response.forEach((item) => - returnItems.push({ - json: item, - pairedItem: { - item: itemIndex, - }, - }), - ); + // eslint-disable-next-line prefer-const + for (let [index, response] of Object.entries(responses)) { + if (this.getMode() === 'manual' && index === '0') { + // For manual executions save the first response in the context + // so that we can use it in the frontend and so make it easier for + // the users to create the required pagination expressions + const nodeContext = this.getContext('node'); + if (pagination && pagination.paginationMode !== 'off') { + nodeContext.response = responseData.value[0]; } else { + nodeContext.response = responseData.value; + } + } + + if (autoDetectResponseFormat) { + const responseContentType = response.headers['content-type'] ?? ''; + if (responseContentType.includes('application/json')) { + responseFormat = 'json'; + if (!response.__bodyResolved) { + const neverError = this.getNodeParameter( + 'options.response.response.neverError', + 0, + false, + ) as boolean; + + const data = await this.helpers + .binaryToBuffer(response.body as Buffer | Readable) + .then((body) => body.toString()); + response.body = jsonParse(data, { + ...(neverError + ? { fallbackValue: {} } + : { errorMessage: 'Invalid JSON in response body' }), + }); + } + } else if (binaryContentTypes.some((e) => responseContentType.includes(e))) { + responseFormat = 'file'; + } else { + responseFormat = 'text'; + if (!response.__bodyResolved) { + const data = await this.helpers + .binaryToBuffer(response.body as Buffer | Readable) + .then((body) => body.toString()); + response.body = !data ? undefined : data; + } + } + } + + if (autoDetectResponseFormat && !fullResponse) { + delete response.headers; + delete response.statusCode; + delete response.statusMessage; + } + if (!fullResponse) { + response = response.body; + } + + if (responseFormat === 'file') { + const outputPropertyName = this.getNodeParameter( + 'options.response.response.outputPropertyName', + 0, + 'data', + ) as string; + + const newItem: INodeExecutionData = { + json: {}, + binary: {}, + pairedItem: { + item: itemIndex, + }, + }; + + if (items[itemIndex].binary !== undefined) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary); + } + + let binaryData: Buffer | Readable; + if (fullResponse) { + const returnItem: IDataObject = {}; + for (const property of fullResponseProperties) { + if (property === 'body') { + continue; + } + returnItem[property] = response[property]; + } + + newItem.json = returnItem; + binaryData = response?.body; + } else { + newItem.json = items[itemIndex].json; + binaryData = response; + } + newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(binaryData); + + returnItems.push(newItem); + } else if (responseFormat === 'text') { + const outputPropertyName = this.getNodeParameter( + 'options.response.response.outputPropertyName', + 0, + 'data', + ) as string; + if (fullResponse) { + const returnItem: IDataObject = {}; + for (const property of fullResponseProperties) { + if (property === 'body') { + returnItem[outputPropertyName] = toText(response[property]); + continue; + } + + returnItem[property] = response[property]; + } returnItems.push({ - json: response, + json: returnItem, pairedItem: { item: itemIndex, }, }); + } else { + returnItems.push({ + json: { + [outputPropertyName]: toText(response), + }, + pairedItem: { + item: itemIndex, + }, + }); + } + } else { + // responseFormat: 'json' + if (fullResponse) { + const returnItem: IDataObject = {}; + for (const property of fullResponseProperties) { + returnItem[property] = response[property]; + } + + if (responseFormat === 'json' && typeof returnItem.body === 'string') { + try { + returnItem.body = JSON.parse(returnItem.body); + } catch (error) { + throw new NodeOperationError( + this.getNode(), + 'Response body is not valid JSON. Change "Response Format" to "Text"', + { itemIndex }, + ); + } + } + + returnItems.push({ + json: returnItem, + pairedItem: { + item: itemIndex, + }, + }); + } else { + if (responseFormat === 'json' && typeof response === 'string') { + try { + if (typeof response !== 'object') { + response = JSON.parse(response); + } + } catch (error) { + throw new NodeOperationError( + this.getNode(), + 'Response body is not valid JSON. Change "Response Format" to "Text"', + { itemIndex }, + ); + } + } + + if (Array.isArray(response)) { + // eslint-disable-next-line @typescript-eslint/no-loop-func + response.forEach((item) => + returnItems.push({ + json: item, + pairedItem: { + item: itemIndex, + }, + }), + ); + } else { + returnItems.push({ + json: response, + pairedItem: { + item: itemIndex, + }, + }); + } } } } diff --git a/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts b/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts index fdd31170d4..b484689109 100644 --- a/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts +++ b/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts @@ -6,6 +6,7 @@ import { workflowToTests, getWorkflowFilenames, } from '@test/nodes/Helpers'; +import { parse as parseUrl } from 'url'; describe('Test HTTP Request Node', () => { const workflows = getWorkflowFilenames(__dirname); @@ -117,6 +118,48 @@ describe('Test HTTP Request Node', () => { isDeleted: true, deletedOn: '2023-02-09T05:37:31.720Z', }); + + // Pagination - Data not identical to dummyjson.com + nock(baseUrl) + .persist() + .get('/users') + .query(true) + .reply(function (uri) { + const data = parseUrl(uri, true); + const skip = parseInt((data.query.skip as string) || '0', 10); + const limit = parseInt((data.query.limit as string) || '10', 10); + const nextUrl = `${baseUrl}/users?skip=${skip + limit}&limit=${limit}`; + + const response = []; + for (let i = skip; i < skip + limit; i++) { + if (i > 14) { + break; + } + response.push({ + id: i, + }); + } + + if (!response.length) { + return [ + 404, + response, + { + 'next-url': nextUrl, + 'content-type': this.req.headers['content-type'] || 'application/json', + }, + ]; + } + + return [ + 200, + response, + { + 'next-url': nextUrl, + 'content-type': this.req.headers['content-type'] || 'application/json', + }, + ]; + }); }); afterAll(() => { diff --git a/packages/nodes-base/nodes/HttpRequest/test/node/workflow.pagination.json b/packages/nodes-base/nodes/HttpRequest/test/node/workflow.pagination.json new file mode 100644 index 0000000000..bf3fce55e4 --- /dev/null +++ b/packages/nodes-base/nodes/HttpRequest/test/node/workflow.pagination.json @@ -0,0 +1,1537 @@ +{ + "name": "HTTP Pagination Test", + "nodes": [ + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "skip", + "value": "={{ $pageCount * 3 }}" + } + ] + }, + "limitPagesFetched": true, + "maxRequests": 3 + } + } + } + }, + "id": "062086e5-e4c9-4ef2-b506-408167e1f0a5", + "name": "Page Limit", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 0] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "skip", + "value": "={{ $pageCount * 3 }}" + } + ] + } + } + } + } + }, + "id": "5b82121f-ec6f-4638-abc9-2c9d6cd29f41", + "name": "Response Empty", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 180] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "skip", + "value": "={{ $pageCount * 3 }}" + } + ] + }, + "paginationCompleteWhen": "receiveSpecificStatusCodes", + "statusCodesWhenComplete": "404", + "limitPagesFetched": true + } + } + } + }, + "id": "4f794c3e-7f08-4d6f-8907-1216d2bea416", + "name": "Receive Status Code", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 340] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "skip", + "value": "={{ $pageCount * 3 }}" + } + ] + }, + "paginationCompleteWhen": "other", + "completeExpression": "={{ $response.statusCode === 404 }}" + } + } + } + }, + "id": "b0348541-a416-4a54-8ece-3a623b7b6143", + "name": "Complete Expression", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 520] + }, + { + "parameters": { + "content": "### Next URL\nResponse Format: JSON", + "height": 223.6542431762359, + "width": 365.5274479049966 + }, + "id": "d028fb00-eac0-495e-b0a7-6a47958bf293", + "name": "Sticky Note", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [840, 720] + }, + { + "parameters": { + "content": "### Update a Parameter in Each Request\nResponse Format: JSON", + "height": 764.5257091080099, + "width": 354.2110090941684 + }, + "id": "2f48dfae-a194-4d50-9f62-e30768bc3d47", + "name": "Sticky Note1", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [847.3233251176339, -78.47327800785297] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "4" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "text" + } + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "paginationCompleteWhen": "receiveSpecificStatusCodes", + "statusCodesWhenComplete": "404" + } + } + } + }, + "id": "a53f6a9d-945f-4891-8bda-15651dc67267", + "name": "Response Empty - Text", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 1040] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "4" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "limitPagesFetched": true, + "maxRequests": 3 + } + } + } + }, + "id": "698733fb-0a96-4aea-b249-3ecc27147232", + "name": "Response Empty Next with Max Pages", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 780] + }, + { + "parameters": { + "content": "### Next URL\nResponse Format: Text", + "height": 388.6542431762359, + "width": 363.5274479049966 + }, + "id": "51183fa5-a558-4fdc-9aae-60bd5f9a46e9", + "name": "Sticky Note2", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [840, 980] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "skip", + "value": "={{ $pageCount * 3 }}" + } + ] + }, + "paginationCompleteWhen": "other", + "completeExpression": "={{ $response.statusCode === 404 }}" + } + } + } + }, + "id": "b0bcca37-a24d-4a21-b513-e4cd9ad2e9cd", + "name": "Complete Expression - JSON", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, -280] + }, + { + "parameters": { + "content": "### Update a Parameter in Each Request\nResponse Format: JSON", + "height": 232.15942469988397, + "width": 323.21100909416833 + }, + "id": "74ab84c6-4e01-4d02-bdbc-e0b1a8310eb2", + "name": "Sticky Note3", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [840, -347.6337155918741] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "4" + } + ] + }, + "options": { + "response": { + "response": { + "fullResponse": true, + "neverError": true, + "responseFormat": "text" + } + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "paginationCompleteWhen": "receiveSpecificStatusCodes", + "statusCodesWhenComplete": "404" + } + } + } + }, + "id": "be7064ff-518f-4566-bd58-e588d863172d", + "name": "Response Empty - Include Full Response", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 1220] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "paginationMode": "off" + } + } + } + }, + "id": "7102d906-c9b7-42c8-a95d-bf277cc88b51", + "name": "Pagination Off", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 1440] + }, + { + "parameters": { + "content": "### Pagination Off", + "height": 373, + "width": 363 + }, + "id": "34fd6eb1-c480-4336-825c-062849a0c9e4", + "name": "Sticky Note4", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [840, 1400] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true, + "responseFormat": "json" + } + } + } + }, + "id": "d17b4ef6-b603-4c2d-a313-98f4ee0bb610", + "name": "Pagination Not Set", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 1620] + }, + { + "parameters": { + "content": "### Detect identical responses\nThrow then error", + "height": 232.15942469988397, + "width": 394.89100909416834 + }, + "id": "f349bb03-7ee3-46c1-98e9-a88070e7f53f", + "name": "Sticky Note5", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [840, 1840] + }, + { + "parameters": { + "fields": { + "values": [ + { + "name": "Error", + "stringValue": "={{ $json.error.name }}" + } + ] + }, + "include": "none", + "options": {} + }, + "id": "3bf2681e-0c13-4998-bade-516faa1098a8", + "name": "Edit Fields", + "type": "n8n-nodes-base.set", + "typeVersion": 3.2, + "position": [1080, 1920] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "json" + } + }, + "pagination": { + "pagination": { + "parameters": { + "parameters": [ + { + "name": "does_not_matter", + "value": "0" + } + ] + } + } + } + } + }, + "id": "c82cbdd8-92c6-43c8-9e27-da9f6106035e", + "name": "Loop", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [900, 1920], + "continueOnFail": true + }, + { + "parameters": { + "content": "### Next URL\nResponse Format: Autodetect\nActual Response Format: JSON", + "height": 458.3224664750446, + "width": 323.21100909416833 + }, + "id": "d0f3ef25-fdab-4d05-a913-9ac93d0c4b9c", + "name": "Sticky Note6", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [820, -1280] + }, + { + "parameters": { + "content": "# Response Format: Autodetect\n", + "width": 545.8929725020898 + }, + "id": "f4c57fc9-8cc4-44ee-a4e6-73b2fa24f4d7", + "name": "Sticky Note7", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [200, -1272] + }, + { + "parameters": { + "content": "# Response Format: set", + "width": 545.8929725020898 + }, + "id": "368400f3-3e43-4f0c-9d4b-b5e01e7f91a5", + "name": "Sticky Note8", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [220, -340] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "response": { + "response": {} + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "limitPagesFetched": true, + "maxRequests": 2 + } + } + } + }, + "id": "e4bf07be-251f-4f65-88f8-50a247530ca1", + "name": "Complete Expression - JSON Autodetect set", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [880, -1192] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "3" + } + ] + }, + "options": { + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "limitPagesFetched": true, + "maxRequests": 2 + } + } + } + }, + "id": "b8f606c8-c559-4616-8760-b110b6dbecda", + "name": "Complete Expression - JSON unset", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [880, -1012] + }, + { + "parameters": {}, + "id": "bd5ecfef-05d2-4b09-906d-9989f162ecfa", + "name": "No Operation, do nothing1", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [420, 520] + }, + { + "parameters": {}, + "id": "2be55c5f-3986-4631-bb9e-54a1f87e95b9", + "name": "Data 2", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [120, 0] + }, + { + "parameters": {}, + "id": "0e5cdaf4-5766-4c0c-b89b-790951caa9b2", + "name": "Data 1", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [440, -1020] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "4" + } + ] + }, + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "content-type", + "value": "text/plain" + } + ] + }, + "options": { + "response": { + "response": { + "neverError": true + } + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "paginationCompleteWhen": "receiveSpecificStatusCodes", + "statusCodesWhenComplete": "404" + } + } + } + }, + "id": "d348ab0a-99c8-4151-bd15-fad4a247834e", + "name": "Response Empty - Text1", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [880, -720] + }, + { + "parameters": { + "content": "### Next URL\nResponse Format: Autodetect\nActual Response Format: text", + "height": 437.60980047313967, + "width": 323.31395441111135 + }, + "id": "4be6d042-ee89-45da-bcac-07f9c4ce4dac", + "name": "Sticky Note9", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [820, -806.2261914090556] + }, + { + "parameters": { + "url": "https://dummyjson.com/users", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "limit", + "value": "4" + } + ] + }, + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "content-type", + "value": "text/plain" + } + ] + }, + "options": { + "response": { + "response": { + "fullResponse": true, + "neverError": true + } + }, + "pagination": { + "pagination": { + "paginationMode": "responseContainsNextURL", + "nextURL": "={{ $response.headers[\"next-url\"] }}", + "paginationCompleteWhen": "receiveSpecificStatusCodes", + "statusCodesWhenComplete": "404" + } + } + } + }, + "id": "5b482293-5176-4408-a56f-036a2a8b16e8", + "name": "Response Empty - Include Full Response1", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.1, + "position": [880, -540] + } + ], + "pinData": { + "Page Limit": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + } + ], + "Response Empty": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + }, + { + "json": { + "id": 9 + } + }, + { + "json": { + "id": 10 + } + }, + { + "json": { + "id": 11 + } + }, + { + "json": { + "id": 12 + } + }, + { + "json": { + "id": 13 + } + }, + { + "json": { + "id": 14 + } + } + ], + "Receive Status Code": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + }, + { + "json": { + "id": 9 + } + }, + { + "json": { + "id": 10 + } + }, + { + "json": { + "id": 11 + } + }, + { + "json": { + "id": 12 + } + }, + { + "json": { + "id": 13 + } + }, + { + "json": { + "id": 14 + } + } + ], + "Complete Expression": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + }, + { + "json": { + "id": 9 + } + }, + { + "json": { + "id": 10 + } + }, + { + "json": { + "id": 11 + } + }, + { + "json": { + "id": 12 + } + }, + { + "json": { + "id": 13 + } + }, + { + "json": { + "id": 14 + } + } + ], + "Response Empty - Text": [ + { + "json": { + "data": "[{\"id\":0},{\"id\":1},{\"id\":2},{\"id\":3}]" + } + }, + { + "json": { + "data": "[{\"id\":4},{\"id\":5},{\"id\":6},{\"id\":7}]" + } + }, + { + "json": { + "data": "[{\"id\":8},{\"id\":9},{\"id\":10},{\"id\":11}]" + } + }, + { + "json": { + "data": "[{\"id\":12},{\"id\":13},{\"id\":14}]" + } + }, + { + "json": { + "data": "[]" + } + } + ], + "Response Empty Next with Max Pages": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + }, + { + "json": { + "id": 9 + } + }, + { + "json": { + "id": 10 + } + }, + { + "json": { + "id": 11 + } + } + ], + "Response Empty - Include Full Response": [ + { + "json": { + "data": "[{\"id\":0},{\"id\":1},{\"id\":2},{\"id\":3}]", + "headers": { + "content-type": "application/json", + "next-url": "https://dummyjson.com/users?skip=4&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":4},{\"id\":5},{\"id\":6},{\"id\":7}]", + "headers": { + "content-type": "application/json", + "next-url": "https://dummyjson.com/users?skip=8&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":8},{\"id\":9},{\"id\":10},{\"id\":11}]", + "headers": { + "content-type": "application/json", + "next-url": "https://dummyjson.com/users?skip=12&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":12},{\"id\":13},{\"id\":14}]", + "headers": { + "content-type": "application/json", + "next-url": "https://dummyjson.com/users?skip=16&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[]", + "headers": { + "content-type": "application/json", + "next-url": "https://dummyjson.com/users?skip=20&limit=4" + }, + "statusCode": 404, + "statusMessage": null + } + } + ], + "Complete Expression - JSON": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + }, + { + "json": { + "id": 6 + } + }, + { + "json": { + "id": 7 + } + }, + { + "json": { + "id": 8 + } + }, + { + "json": { + "id": 9 + } + }, + { + "json": { + "id": 10 + } + }, + { + "json": { + "id": 11 + } + }, + { + "json": { + "id": 12 + } + }, + { + "json": { + "id": 13 + } + }, + { + "json": { + "id": 14 + } + } + ], + "Pagination Off": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + } + ], + "Pagination Not Set": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + } + ], + "Edit Fields": [ + { + "json": { + "Error": "NodeOperationError" + } + } + ], + "Complete Expression - JSON Autodetect set": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + } + ], + "Complete Expression - JSON unset": [ + { + "json": { + "id": 0 + } + }, + { + "json": { + "id": 1 + } + }, + { + "json": { + "id": 2 + } + }, + { + "json": { + "id": 3 + } + }, + { + "json": { + "id": 4 + } + }, + { + "json": { + "id": 5 + } + } + ], + "Response Empty - Text1": [ + { + "json": { + "data": "[{\"id\":0},{\"id\":1},{\"id\":2},{\"id\":3}]" + } + }, + { + "json": { + "data": "[{\"id\":4},{\"id\":5},{\"id\":6},{\"id\":7}]" + } + }, + { + "json": { + "data": "[{\"id\":8},{\"id\":9},{\"id\":10},{\"id\":11}]" + } + }, + { + "json": { + "data": "[{\"id\":12},{\"id\":13},{\"id\":14}]" + } + }, + { + "json": { + "data": "[]" + } + } + ], + "Response Empty - Include Full Response1": [ + { + "json": { + "data": "[{\"id\":0},{\"id\":1},{\"id\":2},{\"id\":3}]", + "headers": { + "content-type": "text/plain", + "next-url": "https://dummyjson.com/users?skip=4&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":4},{\"id\":5},{\"id\":6},{\"id\":7}]", + "headers": { + "content-type": "text/plain", + "next-url": "https://dummyjson.com/users?skip=8&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":8},{\"id\":9},{\"id\":10},{\"id\":11}]", + "headers": { + "content-type": "text/plain", + "next-url": "https://dummyjson.com/users?skip=12&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[{\"id\":12},{\"id\":13},{\"id\":14}]", + "headers": { + "content-type": "text/plain", + "next-url": "https://dummyjson.com/users?skip=16&limit=4" + }, + "statusCode": 200, + "statusMessage": null + } + }, + { + "json": { + "data": "[]", + "headers": { + "content-type": "text/plain", + "next-url": "https://dummyjson.com/users?skip=20&limit=4" + }, + "statusCode": 404, + "statusMessage": null + } + } + ] + }, + "connections": { + "Loop": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "No Operation, do nothing1": { + "main": [ + [ + { + "node": "Receive Status Code", + "type": "main", + "index": 0 + }, + { + "node": "Complete Expression", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty Next with Max Pages", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty - Text", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty - Include Full Response", + "type": "main", + "index": 0 + }, + { + "node": "Pagination Off", + "type": "main", + "index": 0 + }, + { + "node": "Pagination Not Set", + "type": "main", + "index": 0 + }, + { + "node": "Loop", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty", + "type": "main", + "index": 0 + }, + { + "node": "Page Limit", + "type": "main", + "index": 0 + }, + { + "node": "Complete Expression - JSON", + "type": "main", + "index": 0 + } + ] + ] + }, + "Data 2": { + "main": [ + [ + { + "node": "Data 1", + "type": "main", + "index": 0 + }, + { + "node": "No Operation, do nothing1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Data 1": { + "main": [ + [ + { + "node": "Complete Expression - JSON Autodetect set", + "type": "main", + "index": 0 + }, + { + "node": "Complete Expression - JSON unset", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty - Text1", + "type": "main", + "index": 0 + }, + { + "node": "Response Empty - Include Full Response1", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "versionId": "24c0feef-1565-4936-a28f-7111760e6204", + "id": "ZKVF9JrrzoxXbUqu", + "meta": { + "instanceId": "021d3c82ba2d3bc090cbf4fc81c9312668bcc34297e022bb3438c5c88a43a5ff" + }, + "tags": [] +} diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index fcc85ede86..8376930378 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -511,10 +511,18 @@ export interface IHttpRequestOptions { json?: boolean; } +export interface PaginationOptions { + binaryResult?: boolean; + continue: boolean | string; + request: IRequestOptionsSimplifiedAuth; + maxRequests?: number; +} + export type IN8nHttpResponse = IDataObject | Buffer | GenericValue | GenericValue[] | null; export interface IN8nHttpFullResponse { body: IN8nHttpResponse | Readable; + __bodyResolved?: boolean; headers: IDataObject; statusCode: number; statusMessage?: string; @@ -708,6 +716,14 @@ export interface RequestHelperFunctions { requestOptions: IHttpRequestOptions, additionalCredentialOptions?: IAdditionalCredentialOptions, ): Promise; + requestWithAuthenticationPaginated( + this: IAllExecuteFunctions, + requestOptions: OptionsWithUri, + itemIndex: number, + paginationOptions: PaginationOptions, + credentialsType?: string, + additionalCredentialOptions?: IAdditionalCredentialOptions, + ): Promise; requestOAuth1( this: IAllExecuteFunctions, @@ -745,10 +761,12 @@ type FunctionsBaseWithRequiredKeys = Functions [K in Keys]: NonNullable; }; +export type ContextType = 'flow' | 'node'; + type BaseExecutionFunctions = FunctionsBaseWithRequiredKeys<'getMode'> & { continueOnFail(): boolean; evaluateExpression(expression: string, itemIndex: number): NodeParameterValueType; - getContext(type: string): IContextObject; + getContext(type: ContextType): IContextObject; getExecuteData(): IExecuteData; getWorkflowDataProxy(itemIndex: number): IWorkflowDataProxyData; getInputSourceData(inputIndex?: number, inputName?: string): ISourceData;