feat(HTTP Request Node): Add pagination support (#5993)

Is still WIP and does not implement the correct UI yet.

Github issue / Community forum post (link here to close automatically):
https://community.n8n.io/t/pagination-included-into-http-node/15080

https://community.n8n.io/t/how-to-paginate-through-data-in-http-requests/28103

packages/core/src/Constants.ts

@@ -1,6 +1,7 @@
 export const CUSTOM_EXTENSION_ENV = 'N8N_CUSTOM_EXTENSIONS';
 export const PLACEHOLDER_EMPTY_EXECUTION_ID = '__UNKNOWN__';
 export const PLACEHOLDER_EMPTY_WORKFLOW_ID = '__EMPTY__';
+export const HTTP_REQUEST_NODE_TYPE = 'n8n-nodes-base.httpRequest';
 
 export const CUSTOM_NODES_CATEGORY = 'Custom Nodes';
 

packages/core/src/NodeExecuteFunctions.ts

@@ -37,6 +37,7 @@ import { extension, lookup } from 'mime-types';
 import type {
 	BinaryHelperFunctions,
 	ConnectionTypes,
+	ContextType,
 	ExecutionError,
 	FieldType,
 	FileSystemHelperFunctions,
@@ -88,6 +89,7 @@ import type {
 	NodeExecutionWithMetadata,
 	NodeHelperFunctions,
 	NodeParameterValueType,
+	PaginationOptions,
 	RequestHelperFunctions,
 	Workflow,
 	WorkflowActivateMode,
@@ -110,13 +112,14 @@ import {
 	isResourceMapperValue,
 	validateFieldType,
 	ExecutionBaseError,
+	jsonParse,
 } from 'n8n-workflow';
 import type { Token } from 'oauth-1.0a';
 import clientOAuth1 from 'oauth-1.0a';
 import path from 'path';
 import { stringify } from 'qs';
-import type { OptionsWithUri, OptionsWithUrl } from 'request';
-import type { RequestPromiseOptions } from 'request-promise-native';
+import type { OptionsWithUrl } from 'request';
+import type { OptionsWithUri, RequestPromiseOptions } from 'request-promise-native';
 import { Readable } from 'stream';
 import url, { URL, URLSearchParams } from 'url';
 
@@ -126,6 +129,7 @@ import {
 	BLOCK_FILE_ACCESS_TO_N8N_FILES,
 	CONFIG_FILES,
 	CUSTOM_EXTENSION_ENV,
+	HTTP_REQUEST_NODE_TYPE,
 	PLACEHOLDER_EMPTY_EXECUTION_ID,
 	RESTRICT_FILE_ACCESS_TO,
 	UM_EMAIL_TEMPLATES_INVITE,
@@ -143,6 +147,7 @@ import {
 import { getSecretsProxy } from './Secrets';
 import Container from 'typedi';
 import type { BinaryData } from './BinaryData/types';
+import merge from 'lodash/merge';
 import { InstanceSettings } from './InstanceSettings';
 
 axios.defaults.timeout = 300000;
@@ -1866,7 +1871,7 @@ export async function getCredentials(
 
 	// Hardcode for now for security reasons that only a single node can access
 	// all credentials
-	const fullAccess = ['n8n-nodes-base.httpRequest'].includes(node.type);
+	const fullAccess = [HTTP_REQUEST_NODE_TYPE].includes(node.type);
 
 	let nodeCredentialDescription: INodeCredentialDescription | undefined;
 	if (!fullAccess) {
@@ -2239,6 +2244,7 @@ export function getNodeParameter(
 	}
 
 	let returnData;
+
 	try {
 		returnData = workflow.expression.getParameterValue(
 			value,
@@ -2506,9 +2512,241 @@ const getRequestHelperFunctions = (
 	workflow: Workflow,
 	node: INode,
 	additionalData: IWorkflowExecuteAdditionalData,
-): RequestHelperFunctions => ({
-	httpRequest,
+): RequestHelperFunctions => {
+	const getResolvedValue = (
+		parameterValue: NodeParameterValueType,
+		itemIndex: number,
+		runIndex: number,
+		executeData: IExecuteData,
+		additionalKeys?: IWorkflowDataProxyAdditionalKeys,
+		returnObjectAsString = false,
+	): NodeParameterValueType => {
+		const runExecutionData: IRunExecutionData | null = null;
+		const connectionInputData: INodeExecutionData[] = [];
+		const mode: WorkflowExecuteMode = 'internal';
 
+		if (
+			typeof parameterValue === 'object' ||
+			(typeof parameterValue === 'string' && parameterValue.charAt(0) === '=')
+		) {
+			return workflow.expression.getParameterValue(
+				parameterValue,
+				runExecutionData,
+				runIndex,
+				itemIndex,
+				node.name,
+				connectionInputData,
+				mode,
+				additionalKeys ?? {},
+				executeData,
+				returnObjectAsString,
+			);
+		}
+
+		return parameterValue;
+	};
+
+	return {
+		httpRequest,
+		async requestWithAuthenticationPaginated(
+			this: IExecuteFunctions,
+			requestOptions: OptionsWithUri,
+			itemIndex: number,
+			paginationOptions: PaginationOptions,
+			credentialsType?: string,
+			additionalCredentialOptions?: IAdditionalCredentialOptions,
+		): Promise<any[]> {
+			const responseData = [];
+			if (!requestOptions.qs) {
+				requestOptions.qs = {};
+			}
+			requestOptions.resolveWithFullResponse = true;
+			requestOptions.simple = false;
+
+			let tempResponseData: IN8nHttpFullResponse;
+			let makeAdditionalRequest: boolean;
+			let paginateRequestData: IHttpRequestOptions;
+
+			const runIndex = 0;
+
+			const additionalKeys = {
+				$request: requestOptions,
+				$response: {} as IN8nHttpFullResponse,
+				$version: node.typeVersion,
+				$pageCount: 0,
+			};
+
+			const executeData: IExecuteData = {
+				data: {},
+				node,
+				source: null,
+			};
+
+			const hashData = {
+				identicalCount: 0,
+				previousLength: 0,
+				previousHash: '',
+			};
+			do {
+				paginateRequestData = getResolvedValue(
+					paginationOptions.request as unknown as NodeParameterValueType,
+					itemIndex,
+					runIndex,
+					executeData,
+					additionalKeys,
+					false,
+				) as object as IHttpRequestOptions;
+
+				const tempRequestOptions = merge(requestOptions, paginateRequestData);
+
+				if (credentialsType) {
+					tempResponseData = await this.helpers.requestWithAuthentication.call(
+						this,
+						credentialsType,
+						tempRequestOptions,
+						additionalCredentialOptions,
+					);
+				} else {
+					tempResponseData = await this.helpers.request(tempRequestOptions);
+				}
+
+				const newResponse: IN8nHttpFullResponse = Object.assign(
+					{
+						body: {},
+						headers: {},
+						statusCode: 0,
+					},
+					pick(tempResponseData, ['body', 'headers', 'statusCode']),
+				);
+
+				let contentBody: Exclude<IN8nHttpResponse, Buffer>;
+
+				if (
+					newResponse.body?.constructor.name === 'IncomingMessage' &&
+					paginationOptions.binaryResult !== true
+				) {
+					const data = await this.helpers
+						.binaryToBuffer(newResponse.body as Buffer | Readable)
+						.then((body) => body.toString());
+					// Keep the original string version that we can use it to hash if needed
+					contentBody = data;
+
+					const responseContentType = newResponse.headers['content-type']?.toString() ?? '';
+					if (responseContentType.includes('application/json')) {
+						newResponse.body = jsonParse(data, { fallbackValue: {} });
+					} else {
+						newResponse.body = data;
+					}
+					tempResponseData.__bodyResolved = true;
+					tempResponseData.body = newResponse.body;
+				} else {
+					contentBody = newResponse.body;
+				}
+
+				if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) {
+					// If the data is not binary (and so not a stream), or an etag is present,
+					// we check via etag or hash if identical data is received
+
+					let contentLength = 0;
+					if ('content-length' in tempResponseData.headers) {
+						contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0;
+					}
+
+					if (hashData.previousLength === contentLength) {
+						let hash: string;
+						if (tempResponseData.headers.etag) {
+							// If an etag is provided, we use it as "hash"
+							hash = tempResponseData.headers.etag as string;
+						} else {
+							// If there is no etag, we calculate a hash from the data in the body
+							if (typeof contentBody !== 'string') {
+								contentBody = JSON.stringify(contentBody);
+							}
+							hash = crypto.createHash('md5').update(contentBody).digest('base64');
+						}
+
+						if (hashData.previousHash === hash) {
+							hashData.identicalCount += 1;
+							if (hashData.identicalCount > 2) {
+								// Length was identical 5x and hash 3x
+								throw new NodeOperationError(
+									node,
+									'The returned response was identical 5x, so requests got stopped',
+									{
+										itemIndex,
+										description:
+											'Check if "Pagination Completed When" has been configured correctly.',
+									},
+								);
+							}
+						} else {
+							hashData.identicalCount = 0;
+						}
+						hashData.previousHash = hash;
+					} else {
+						hashData.identicalCount = 0;
+					}
+					hashData.previousLength = contentLength;
+				}
+
+				responseData.push(tempResponseData);
+
+				additionalKeys.$response = newResponse;
+				additionalKeys.$pageCount = additionalKeys.$pageCount + 1;
+
+				if (
+					paginationOptions.maxRequests &&
+					additionalKeys.$pageCount >= paginationOptions.maxRequests
+				) {
+					break;
+				}
+
+				makeAdditionalRequest = getResolvedValue(
+					paginationOptions.continue,
+					itemIndex,
+					runIndex,
+					executeData,
+					additionalKeys,
+					false,
+				) as boolean;
+
+				if (makeAdditionalRequest) {
+					if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) {
+						// We have it configured to let all requests pass no matter the response code
+						// via "requestOptions.simple = false" to not by default fail if it is for example
+						// configured to stop on 404 response codes. For that reason we have to throw here
+						// now an error manually if the response code is not a success one.
+						let data = tempResponseData.body;
+						if (
+							data?.constructor.name === 'IncomingMessage' &&
+							paginationOptions.binaryResult !== true
+						) {
+							data = await this.helpers
+								.binaryToBuffer(tempResponseData.body as Buffer | Readable)
+								.then((body) => body.toString());
+						} else if (typeof data === 'object') {
+							data = JSON.stringify(data);
+						}
+
+						throw Object.assign(
+							new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`),
+							{
+								statusCode: tempResponseData.statusCode,
+								error: data,
+								isAxiosError: true,
+								response: {
+									headers: tempResponseData.headers,
+									status: tempResponseData.statusCode,
+									statusText: tempResponseData.statusMessage,
+								},
+							},
+						);
+					}
+				}
+			} while (makeAdditionalRequest);
+
+			return responseData;
+		},
 		async httpRequestWithAuthentication(
 			this,
 			credentialsType,
@@ -2569,7 +2807,8 @@ const getRequestHelperFunctions = (
 				oAuth2Options,
 			);
 		},
-});
+	};
+};
 
 const getAllowedPaths = () => {
 	const restrictFileAccessTo = process.env[RESTRICT_FILE_ACCESS_TO];
@@ -2899,7 +3138,7 @@ export function getExecuteFunctions(
 						),
 					);
 			},
-			getContext(type: string): IContextObject {
+			getContext(type: ContextType): IContextObject {
 				return NodeHelpers.getContext(runExecutionData, type, node);
 			},
 			async getInputConnectionData(
@@ -3293,7 +3532,7 @@ export function getExecuteSingleFunctions(
 					executeData,
 				);
 			},
-			getContext(type: string): IContextObject {
+			getContext(type: ContextType): IContextObject {
 				return NodeHelpers.getContext(runExecutionData, type, node);
 			},
 			getCredentials: async (type) =>

packages/editor-ui/src/mixins/workflowHelpers.ts

@@ -1,5 +1,6 @@
 import {
 	EnterpriseEditionFeature,
+	HTTP_REQUEST_NODE_TYPE,
 	MODAL_CONFIRM,
 	PLACEHOLDER_EMPTY_WORKFLOW_ID,
 	PLACEHOLDER_FILLED_AT_EXECUTION_TIME,
@@ -49,7 +50,7 @@ import { externalHooks } from '@/mixins/externalHooks';
 import { genericHelpers } from '@/mixins/genericHelpers';
 import { nodeHelpers } from '@/mixins/nodeHelpers';
 
-import { isEqual } from 'lodash-es';
+import { get, isEqual } from 'lodash-es';
 
 import type { IPermissions } from '@/permissions';
 import { getWorkflowPermissions } from '@/permissions';
@@ -194,6 +195,16 @@ export function resolveParameter(
 		...opts.additionalKeys,
 	};
 
+	if (activeNode?.type === HTTP_REQUEST_NODE_TYPE) {
+		// Add $response for HTTP Request-Nodes as it is used
+		// in pagination expressions
+		additionalKeys.$response = get(
+			executionData,
+			`data.executionData.contextData['node:${activeNode!.name}'].response`,
+			{},
+		);
+	}
+
 	let runIndexCurrent = opts?.targetItem?.runIndex ?? 0;
 	if (
 		opts?.targetItem === undefined &&

packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts

@@ -9,6 +9,7 @@ import type {
 	INodeTypeBaseDescription,
 	INodeTypeDescription,
 	IRequestOptionsSimplified,
+	PaginationOptions,
 	JsonObject,
 } from 'n8n-workflow';
 
@@ -927,6 +928,228 @@ export class HttpRequestV3 implements INodeType {
 								},
 							],
 						},
+						{
+							displayName: 'Pagination',
+							name: 'pagination',
+							placeholder: 'Add pagination',
+							type: 'fixedCollection',
+							typeOptions: {
+								multipleValues: false,
+							},
+							default: {
+								pagination: {},
+							},
+							options: [
+								{
+									displayName: 'Pagination',
+									name: 'pagination',
+									values: [
+										{
+											displayName: 'Pagination Mode',
+											name: 'paginationMode',
+											type: 'options',
+											typeOptions: {
+												noDataExpression: true,
+											},
+											options: [
+												{
+													name: 'Off',
+													value: 'off',
+												},
+												{
+													name: 'Update a Parameter in Each Request',
+													value: 'updateAParameterInEachRequest',
+												},
+												{
+													name: 'Response Contains Next URL',
+													value: 'responseContainsNextURL',
+												},
+											],
+											default: 'updateAParameterInEachRequest',
+											description: 'If pagination should be used',
+										},
+										{
+											displayName:
+												'Use the $response variables to access the data of the previous response. <a href="https://docs.n8n.io/code/builtin/http-node-variables/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.httpRequest" target="_blank">More info</a>',
+											name: 'webhookNotice',
+											displayOptions: {
+												hide: {
+													paginationMode: ['off'],
+												},
+											},
+											type: 'notice',
+											default: '',
+										},
+										{
+											displayName: 'Next URL',
+											name: 'nextURL',
+											type: 'string',
+											displayOptions: {
+												show: {
+													paginationMode: ['responseContainsNextURL'],
+												},
+											},
+											default: '',
+											description:
+												'Should evaluate to true when pagination is complete. More info.',
+										},
+										{
+											displayName: 'Parameters',
+											name: 'parameters',
+											type: 'fixedCollection',
+											displayOptions: {
+												show: {
+													paginationMode: ['updateAParameterInEachRequest'],
+												},
+											},
+											typeOptions: {
+												multipleValues: true,
+												noExpression: true,
+											},
+											placeholder: 'Add Parameter',
+											default: {
+												parameters: [
+													{
+														type: 'qs',
+														name: '',
+														value: '',
+													},
+												],
+											},
+											options: [
+												{
+													name: 'parameters',
+													displayName: 'Parameter',
+													values: [
+														{
+															displayName: 'Type',
+															name: 'type',
+															type: 'options',
+															options: [
+																{
+																	name: 'Body',
+																	value: 'body',
+																},
+																{
+																	name: 'Header',
+																	value: 'headers',
+																},
+																{
+																	name: 'Query',
+																	value: 'qs',
+																},
+															],
+															default: 'qs',
+															description: 'Where the parameter should be set',
+														},
+														{
+															displayName: 'Name',
+															name: 'name',
+															type: 'string',
+															default: '',
+														},
+														{
+															displayName: 'Value',
+															name: 'value',
+															type: 'string',
+															default: '',
+														},
+													],
+												},
+											],
+										},
+										{
+											displayName: 'Pagination Complete When',
+											name: 'paginationCompleteWhen',
+											type: 'options',
+											typeOptions: {
+												noDataExpression: true,
+											},
+											displayOptions: {
+												hide: {
+													paginationMode: ['off'],
+												},
+											},
+											options: [
+												{
+													name: 'Response Is Empty',
+													value: 'responseIsEmpty',
+												},
+												{
+													name: 'Receive Specific Status Code(s)',
+													value: 'receiveSpecificStatusCodes',
+												},
+												{
+													name: 'Other',
+													value: 'other',
+												},
+											],
+											default: 'responseIsEmpty',
+											description: 'When should no further requests be made?',
+										},
+										{
+											displayName: 'Status Code(s) when Complete',
+											name: 'statusCodesWhenComplete',
+											type: 'string',
+											typeOptions: {
+												noDataExpression: true,
+											},
+											displayOptions: {
+												show: {
+													paginationCompleteWhen: ['receiveSpecificStatusCodes'],
+												},
+											},
+											default: '',
+											description: 'Accepts comma-separated values',
+										},
+										{
+											displayName: 'Complete Expression',
+											name: 'completeExpression',
+											type: 'string',
+											displayOptions: {
+												show: {
+													paginationCompleteWhen: ['other'],
+												},
+											},
+											default: '',
+											description:
+												'Should evaluate to true when pagination is complete. More info.',
+										},
+										{
+											displayName: 'Limit Pages Fetched',
+											name: 'limitPagesFetched',
+											type: 'boolean',
+											typeOptions: {
+												noDataExpression: true,
+											},
+											displayOptions: {
+												hide: {
+													paginationMode: ['off'],
+												},
+											},
+											default: false,
+											noDataExpression: true,
+											description: 'Whether the number of requests should be limited',
+										},
+										{
+											displayName: 'Max Pages',
+											name: 'maxRequests',
+											type: 'number',
+											typeOptions: {
+												noDataExpression: true,
+											},
+											displayOptions: {
+												show: {
+													limitPagesFetched: [true],
+												},
+											},
+											default: 100,
+											description: 'Maximum amount of request to be make',
+										},
+									],
+								},
+							],
+						},
 						{
 							displayName: 'Proxy',
 							name: 'proxy',
@@ -1033,6 +1256,26 @@ export class HttpRequestV3 implements INodeType {
 
 		let autoDetectResponseFormat = false;
 
+		// Can not be defined on a per item level
+		const pagination = this.getNodeParameter('options.pagination.pagination', 0, null, {
+			rawExpressions: true,
+		}) as {
+			paginationMode: 'off' | 'updateAParameterInEachRequest' | 'responseContainsNextURL';
+			nextURL?: string;
+			parameters: {
+				parameters: Array<{
+					type: 'body' | 'headers' | 'qs';
+					name: string;
+					value: string;
+				}>;
+			};
+			paginationCompleteWhen: 'responseIsEmpty' | 'receiveSpecificStatusCodes' | 'other';
+			statusCodesWhenComplete: string;
+			completeExpression: string;
+			limitPagesFetched: boolean;
+			maxRequests: number;
+		};
+
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
 			const requestMethod = this.getNodeParameter('method', itemIndex) as string;
 
@@ -1117,15 +1360,9 @@ export class HttpRequestV3 implements INodeType {
 				gzip: true,
 				rejectUnauthorized: !allowUnauthorizedCerts || false,
 				followRedirect: false,
+				resolveWithFullResponse: true,
 			};
 
-			// When response format is set to auto-detect,
-			// we need to access to response header content-type
-			// and the only way is using "resolveWithFullResponse"
-			if (autoDetectResponseFormat || fullResponse) {
-				requestOptions.resolveWithFullResponse = true;
-			}
-
 			if (requestOptions.method !== 'GET' && nodeVersion >= 4.1) {
 				requestOptions = { ...requestOptions, followAllRedirects: false };
 			}
@@ -1321,7 +1558,7 @@ export class HttpRequestV3 implements INodeType {
 				requestOptions.json = true;
 			}
 
-			// // Add Content Type if any are set
+			// Add Content Type if any are set
 			if (bodyContentType === 'raw') {
 				if (requestOptions.headers === undefined) {
 					requestOptions.headers = {};
@@ -1392,7 +1629,64 @@ export class HttpRequestV3 implements INodeType {
 			try {
 				this.sendMessageToUI(sanitizeUiMessage(requestOptions, authDataKeys));
 			} catch (e) {}
-			if (authentication === 'genericCredentialType' || authentication === 'none') {
+
+			if (pagination && pagination.paginationMode !== 'off') {
+				let continueExpression = '={{false}}';
+				if (pagination.paginationCompleteWhen === 'receiveSpecificStatusCodes') {
+					// Split out comma separated list of status codes into array
+					const statusCodesWhenCompleted = pagination.statusCodesWhenComplete
+						.split(',')
+						.map((item) => parseInt(item.trim()));
+
+					continueExpression = `={{ !${JSON.stringify(
+						statusCodesWhenCompleted,
+					)}.includes($response.statusCode) }}`;
+				} else if (pagination.paginationCompleteWhen === 'responseIsEmpty') {
+					continueExpression =
+						'={{ Array.isArray($response.body) ? $response.body.length : !!$response.body }}';
+				} else {
+					// Other
+					if (!pagination.completeExpression.length || pagination.completeExpression[0] !== '=') {
+						throw new NodeOperationError(this.getNode(), 'Invalid or empty Complete Expression');
+					}
+					continueExpression = `={{ !(${pagination.completeExpression.trim().slice(3, -2)}) }}`;
+				}
+
+				const paginationData: PaginationOptions = {
+					continue: continueExpression,
+					request: {},
+				};
+
+				if (pagination.paginationMode === 'updateAParameterInEachRequest') {
+					// Iterate over all parameters and add them to the request
+					paginationData.request = {};
+					pagination.parameters.parameters.forEach((parameter) => {
+						if (!paginationData.request[parameter.type]) {
+							paginationData.request[parameter.type] = {};
+						}
+						paginationData.request[parameter.type]![parameter.name] = parameter.value;
+					});
+				} else if (pagination.paginationMode === 'responseContainsNextURL') {
+					paginationData.request.url = pagination.nextURL;
+				}
+
+				if (pagination.limitPagesFetched) {
+					paginationData.maxRequests = pagination.maxRequests;
+				}
+
+				if (responseFormat === 'file') {
+					paginationData.binaryResult = true;
+				}
+
+				const requestPromise = this.helpers.requestWithAuthenticationPaginated.call(
+					this,
+					requestOptions,
+					itemIndex,
+					paginationData,
+					nodeCredentialType,
+				);
+				requestPromises.push(requestPromise);
+			} else if (authentication === 'genericCredentialType' || authentication === 'none') {
 				if (oAuth1Api) {
 					const requestOAuth1 = this.helpers.requestOAuth1.call(this, 'oAuth1Api', requestOptions);
 					requestOAuth1.catch(() => {});
@@ -1426,25 +1720,25 @@ export class HttpRequestV3 implements INodeType {
 		}
 		const promisesResponses = await Promise.allSettled(requestPromises);
 
-		let response: any;
+		let responseData: any;
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
-			response = promisesResponses.shift();
-			if (response!.status !== 'fulfilled') {
-				if (response.reason.statusCode === 429) {
-					response.reason.message =
+			responseData = promisesResponses.shift();
+			if (responseData!.status !== 'fulfilled') {
+				if (responseData.reason.statusCode === 429) {
+					responseData.reason.message =
 						"Try spacing your requests out using the batching settings under 'Options'";
 				}
 				if (!this.continueOnFail()) {
-					if (autoDetectResponseFormat && response.reason.error instanceof Buffer) {
-						response.reason.error = Buffer.from(response.reason.error as Buffer).toString();
+					if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) {
+						responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString();
 					}
-					throw new NodeApiError(this.getNode(), response as JsonObject, { itemIndex });
+					throw new NodeApiError(this.getNode(), responseData as JsonObject, { itemIndex });
 				} else {
-					removeCircularRefs(response.reason as JsonObject);
+					removeCircularRefs(responseData.reason as JsonObject);
 					// Return the actual reason as error
 					returnItems.push({
 						json: {
-							error: response.reason,
+							error: responseData.reason,
 						},
 						pairedItem: {
 							item: itemIndex,
@@ -1454,7 +1748,12 @@ export class HttpRequestV3 implements INodeType {
 				}
 			}
 
-			response = response.value;
+			let responses: any[];
+			if (Array.isArray(responseData.value)) {
+				responses = responseData.value;
+			} else {
+				responses = [responseData.value];
+			}
 
 			let responseFormat = this.getNodeParameter(
 				'options.response.response.responseFormat',
@@ -1468,10 +1767,25 @@ export class HttpRequestV3 implements INodeType {
 				false,
 			) as boolean;
 
+			// eslint-disable-next-line prefer-const
+			for (let [index, response] of Object.entries(responses)) {
+				if (this.getMode() === 'manual' && index === '0') {
+					// For manual executions save the first response in the context
+					// so that we can use it in the frontend and so make it easier for
+					// the users to create the required pagination expressions
+					const nodeContext = this.getContext('node');
+					if (pagination && pagination.paginationMode !== 'off') {
+						nodeContext.response = responseData.value[0];
+					} else {
+						nodeContext.response = responseData.value;
+					}
+				}
+
 				if (autoDetectResponseFormat) {
 					const responseContentType = response.headers['content-type'] ?? '';
 					if (responseContentType.includes('application/json')) {
 						responseFormat = 'json';
+						if (!response.__bodyResolved) {
 							const neverError = this.getNodeParameter(
 								'options.response.response.neverError',
 								0,
@@ -1486,23 +1800,27 @@ export class HttpRequestV3 implements INodeType {
 									? { fallbackValue: {} }
 									: { errorMessage: 'Invalid JSON in response body' }),
 							});
+						}
 					} else if (binaryContentTypes.some((e) => responseContentType.includes(e))) {
 						responseFormat = 'file';
 					} else {
 						responseFormat = 'text';
+						if (!response.__bodyResolved) {
 							const data = await this.helpers
 								.binaryToBuffer(response.body as Buffer | Readable)
 								.then((body) => body.toString());
 							response.body = !data ? undefined : data;
 						}
 					}
+				}
 
 				if (autoDetectResponseFormat && !fullResponse) {
 					delete response.headers;
 					delete response.statusCode;
 					delete response.statusMessage;
+				}
+				if (!fullResponse) {
 					response = response.body;
-				requestOptions.resolveWithFullResponse = false;
 				}
 
 				if (responseFormat === 'file') {
@@ -1534,11 +1852,11 @@ export class HttpRequestV3 implements INodeType {
 							if (property === 'body') {
 								continue;
 							}
-						returnItem[property] = response![property];
+							returnItem[property] = response[property];
 						}
 
 						newItem.json = returnItem;
-					binaryData = response!.body;
+						binaryData = response?.body;
 					} else {
 						newItem.json = items[itemIndex].json;
 						binaryData = response;
@@ -1556,11 +1874,11 @@ export class HttpRequestV3 implements INodeType {
 						const returnItem: IDataObject = {};
 						for (const property of fullResponseProperties) {
 							if (property === 'body') {
-							returnItem[outputPropertyName] = toText(response![property]);
+								returnItem[outputPropertyName] = toText(response[property]);
 								continue;
 							}
 
-						returnItem[property] = response![property];
+							returnItem[property] = response[property];
 						}
 						returnItems.push({
 							json: returnItem,
@@ -1580,10 +1898,10 @@ export class HttpRequestV3 implements INodeType {
 					}
 				} else {
 					// responseFormat: 'json'
-				if (requestOptions.resolveWithFullResponse === true) {
+					if (fullResponse) {
 						const returnItem: IDataObject = {};
 						for (const property of fullResponseProperties) {
-						returnItem[property] = response![property];
+							returnItem[property] = response[property];
 						}
 
 						if (responseFormat === 'json' && typeof returnItem.body === 'string') {
@@ -1607,7 +1925,9 @@ export class HttpRequestV3 implements INodeType {
 					} else {
 						if (responseFormat === 'json' && typeof response === 'string') {
 							try {
+								if (typeof response !== 'object') {
 									response = JSON.parse(response);
+								}
 							} catch (error) {
 								throw new NodeOperationError(
 									this.getNode(),
@@ -1638,6 +1958,7 @@ export class HttpRequestV3 implements INodeType {
 					}
 				}
 			}
+		}
 
 		returnItems = returnItems.map(replaceNullValues);
 

packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts

@@ -6,6 +6,7 @@ import {
 	workflowToTests,
 	getWorkflowFilenames,
 } from '@test/nodes/Helpers';
+import { parse as parseUrl } from 'url';
 
 describe('Test HTTP Request Node', () => {
 	const workflows = getWorkflowFilenames(__dirname);
@@ -117,6 +118,48 @@ describe('Test HTTP Request Node', () => {
 			isDeleted: true,
 			deletedOn: '2023-02-09T05:37:31.720Z',
 		});
+
+		// Pagination - Data not identical to dummyjson.com
+		nock(baseUrl)
+			.persist()
+			.get('/users')
+			.query(true)
+			.reply(function (uri) {
+				const data = parseUrl(uri, true);
+				const skip = parseInt((data.query.skip as string) || '0', 10);
+				const limit = parseInt((data.query.limit as string) || '10', 10);
+				const nextUrl = `${baseUrl}/users?skip=${skip + limit}&limit=${limit}`;
+
+				const response = [];
+				for (let i = skip; i < skip + limit; i++) {
+					if (i > 14) {
+						break;
+					}
+					response.push({
+						id: i,
+					});
+				}
+
+				if (!response.length) {
+					return [
+						404,
+						response,
+						{
+							'next-url': nextUrl,
+							'content-type': this.req.headers['content-type'] || 'application/json',
+						},
+					];
+				}
+
+				return [
+					200,
+					response,
+					{
+						'next-url': nextUrl,
+						'content-type': this.req.headers['content-type'] || 'application/json',
+					},
+				];
+			});
 	});
 
 	afterAll(() => {

packages/workflow/src/Interfaces.ts

@@ -511,10 +511,18 @@ export interface IHttpRequestOptions {
 	json?: boolean;
 }
 
+export interface PaginationOptions {
+	binaryResult?: boolean;
+	continue: boolean | string;
+	request: IRequestOptionsSimplifiedAuth;
+	maxRequests?: number;
+}
+
 export type IN8nHttpResponse = IDataObject | Buffer | GenericValue | GenericValue[] | null;
 
 export interface IN8nHttpFullResponse {
 	body: IN8nHttpResponse | Readable;
+	__bodyResolved?: boolean;
 	headers: IDataObject;
 	statusCode: number;
 	statusMessage?: string;
@@ -708,6 +716,14 @@ export interface RequestHelperFunctions {
 		requestOptions: IHttpRequestOptions,
 		additionalCredentialOptions?: IAdditionalCredentialOptions,
 	): Promise<any>;
+	requestWithAuthenticationPaginated(
+		this: IAllExecuteFunctions,
+		requestOptions: OptionsWithUri,
+		itemIndex: number,
+		paginationOptions: PaginationOptions,
+		credentialsType?: string,
+		additionalCredentialOptions?: IAdditionalCredentialOptions,
+	): Promise<any[]>;
 
 	requestOAuth1(
 		this: IAllExecuteFunctions,
@@ -745,10 +761,12 @@ type FunctionsBaseWithRequiredKeys<Keys extends keyof FunctionsBase> = Functions
 	[K in Keys]: NonNullable<FunctionsBase[K]>;
 };
 
+export type ContextType = 'flow' | 'node';
+
 type BaseExecutionFunctions = FunctionsBaseWithRequiredKeys<'getMode'> & {
 	continueOnFail(): boolean;
 	evaluateExpression(expression: string, itemIndex: number): NodeParameterValueType;
-	getContext(type: string): IContextObject;
+	getContext(type: ContextType): IContextObject;
 	getExecuteData(): IExecuteData;
 	getWorkflowDataProxy(itemIndex: number): IWorkflowDataProxyData;
 	getInputSourceData(inputIndex?: number, inputName?: string): ISourceData;