fix(HTTP Request Node): Support form data when using pagination (#8497)

packages/core/src/NodeExecuteFunctions.ts

@@ -1208,6 +1208,23 @@ async function prepareBinaryData(
 	return await setBinaryDataBuffer(returnData, binaryData, workflowId, executionId);
 }
 
+function applyPaginationRequestData(
+	requestData: OptionsWithUri,
+	paginationRequestData: PaginationOptions['request'],
+): OptionsWithUri {
+	const preparedPaginationData: Partial<OptionsWithUri> = { ...paginationRequestData };
+
+	if ('formData' in requestData) {
+		preparedPaginationData.formData = paginationRequestData.body;
+		delete preparedPaginationData.body;
+	} else if ('form' in requestData) {
+		preparedPaginationData.form = paginationRequestData.body;
+		delete preparedPaginationData.body;
+	}
+
+	return merge({}, requestData, preparedPaginationData);
+}
+
 /**
  * Makes a request using OAuth data for authentication
  *
@@ -2806,7 +2823,7 @@ const getRequestHelperFunctions = (
 
 			let tempResponseData: IN8nHttpFullResponse;
 			let makeAdditionalRequest: boolean;
-			let paginateRequestData: IHttpRequestOptions;
+			let paginateRequestData: PaginationOptions['request'];
 
 			const runIndex = 0;
 
@@ -2836,9 +2853,9 @@ const getRequestHelperFunctions = (
 					executeData,
 					additionalKeys,
 					false,
-				) as object as IHttpRequestOptions;
+				) as object as PaginationOptions['request'];
 
-				const tempRequestOptions = merge(requestOptions, paginateRequestData);
+				const tempRequestOptions = applyPaginationRequestData(requestOptions, paginateRequestData);
 
 				if (credentialsType) {
 					tempResponseData = await this.helpers.requestWithAuthentication.call(

packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts

@@ -18,6 +18,40 @@ describe('Test HTTP Request Node', () => {
 		await initBinaryDataService();
 		nock.disableNetConnect();
 
+		function getPaginationReturnData(this: nock.ReplyFnContext, limit = 10, skip = 0) {
+			const nextUrl = `${baseUrl}/users?skip=${skip + limit}&limit=${limit}`;
+
+			const response = [];
+			for (let i = skip; i < skip + limit; i++) {
+				if (i > 14) {
+					break;
+				}
+				response.push({
+					id: i,
+				});
+			}
+
+			if (!response.length) {
+				return [
+					404,
+					response,
+					{
+						'next-url': nextUrl,
+						'content-type': this.req.headers['content-type'] || 'application/json',
+					},
+				];
+			}
+
+			return [
+				200,
+				response,
+				{
+					'next-url': nextUrl,
+					'content-type': this.req.headers['content-type'] || 'application/json',
+				},
+			];
+		}
+
 		//GET
 		nock(baseUrl).get('/todos/1').reply(200, {
 			id: 1,
@@ -119,46 +153,36 @@ describe('Test HTTP Request Node', () => {
 			deletedOn: '2023-02-09T05:37:31.720Z',
 		});
 
-		// Pagination - Data not identical to dummyjson.com
+		// Pagination - GET
 		nock(baseUrl)
 			.persist()
 			.get('/users')
 			.query(true)
 			.reply(function (uri) {
 				const data = parseUrl(uri, true);
-				const skip = parseInt((data.query.skip as string) || '0', 10);
 				const limit = parseInt((data.query.limit as string) || '10', 10);
-				const nextUrl = `${baseUrl}/users?skip=${skip + limit}&limit=${limit}`;
+				const skip = parseInt((data.query.skip as string) || '0', 10);
-
+				return getPaginationReturnData.call(this, limit, skip);
-				const response = [];
-				for (let i = skip; i < skip + limit; i++) {
-					if (i > 14) {
-						break;
-					}
-					response.push({
-						id: i,
 			});
+
+		// Pagination - POST
+		nock(baseUrl)
+			.persist()
+			.post('/users')
+			.reply(function (_uri, body) {
+				let skip = 0;
+				let limit = 10;
+
+				if (typeof body === 'string') {
+					// Form data
+					skip = parseInt(body.split('name="skip"')[1].split('---')[0] ?? '0', 10);
+					limit = parseInt(body.split('name="limit"')[1].split('---')[0] ?? '0', 10);
+				} else {
+					skip = parseInt(body.skip ?? '0', 10);
+					limit = parseInt(body.limit ?? '10', 10);
 				}
 
-				if (!response.length) {
+				return getPaginationReturnData.call(this, limit, skip);
-					return [
-						404,
-						response,
-						{
-							'next-url': nextUrl,
-							'content-type': this.req.headers['content-type'] || 'application/json',
-						},
-					];
-				}
-
-				return [
-					200,
-					response,
-					{
-						'next-url': nextUrl,
-						'content-type': this.req.headers['content-type'] || 'application/json',
-					},
-				];
 			});
 	});
 

packages/nodes-base/nodes/HttpRequest/test/node/workflow.pagination.json

@@ -35,11 +35,14 @@
           }
         }
       },
-			"id": "062086e5-e4c9-4ef2-b506-408167e1f0a5",
+      "id": "37742e68-f44e-457a-beb2-f698f4c0dbb5",
       "name": "Page Limit",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 0]
+      "position": [
+        2220,
+        1800
+      ]
     },
     {
       "parameters": {
@@ -74,11 +77,14 @@
           }
         }
       },
-			"id": "5b82121f-ec6f-4638-abc9-2c9d6cd29f41",
+      "id": "07d9001f-5384-4b74-beca-e623467790a8",
       "name": "Response Empty",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 180]
+      "position": [
+        2220,
+        1980
+      ]
     },
     {
       "parameters": {
@@ -115,11 +121,14 @@
           }
         }
       },
-			"id": "4f794c3e-7f08-4d6f-8907-1216d2bea416",
+      "id": "c918b2be-575e-48df-b6bf-36eee64f12c0",
       "name": "Receive Status Code",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 340]
+      "position": [
+        2220,
+        2140
+      ]
     },
     {
       "parameters": {
@@ -156,11 +165,14 @@
           }
         }
       },
-			"id": "b0348541-a416-4a54-8ece-3a623b7b6143",
+      "id": "3ac6da0f-b931-4c0f-9f75-9818e03cb65b",
       "name": "Complete Expression",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 520]
+      "position": [
+        2220,
+        2320
+      ]
     },
     {
       "parameters": {
@@ -168,23 +180,29 @@
         "height": 223.6542431762359,
         "width": 365.5274479049966
       },
-			"id": "d028fb00-eac0-495e-b0a7-6a47958bf293",
+      "id": "29478700-ec0e-4b88-b771-8c12cc17f6e5",
       "name": "Sticky Note",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [840, 720]
+      "position": [
+        2160,
+        2920
+      ]
     },
     {
       "parameters": {
         "content": "### Update a Parameter in Each Request\nResponse Format: JSON",
-				"height": 764.5257091080099,
+        "height": 1140.0832129820226,
         "width": 354.2110090941684
       },
-			"id": "2f48dfae-a194-4d50-9f62-e30768bc3d47",
+      "id": "cf67a13d-6204-4d1d-99c9-dea6eb882a17",
       "name": "Sticky Note1",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [847.3233251176339, -78.47327800785297]
+      "position": [
+        2167,
+        1721
+      ]
     },
     {
       "parameters": {
@@ -215,11 +233,14 @@
           }
         }
       },
-			"id": "a53f6a9d-945f-4891-8bda-15651dc67267",
+      "id": "9c7e0300-e94c-4957-abac-0e6f3f98df94",
       "name": "Response Empty - Text",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 1040]
+      "position": [
+        2220,
+        3240
+      ]
     },
     {
       "parameters": {
@@ -249,11 +270,14 @@
           }
         }
       },
-			"id": "698733fb-0a96-4aea-b249-3ecc27147232",
+      "id": "26d8b9ca-68ca-48c2-b603-93b0a2a9369c",
       "name": "Response Empty Next with Max Pages",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 780]
+      "position": [
+        2220,
+        2980
+      ]
     },
     {
       "parameters": {
@@ -261,11 +285,14 @@
         "height": 388.6542431762359,
         "width": 363.5274479049966
       },
-			"id": "51183fa5-a558-4fdc-9aae-60bd5f9a46e9",
+      "id": "7e71e5db-372b-49f4-a309-fafa8cfa6c42",
       "name": "Sticky Note2",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [840, 980]
+      "position": [
+        2160,
+        3180
+      ]
     },
     {
       "parameters": {
@@ -302,11 +329,14 @@
           }
         }
       },
-			"id": "b0bcca37-a24d-4a21-b513-e4cd9ad2e9cd",
+      "id": "63515259-3498-49f6-86b4-fd0386b61244",
       "name": "Complete Expression - JSON",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, -280]
+      "position": [
+        2220,
+        1520
+      ]
     },
     {
       "parameters": {
@@ -314,11 +344,14 @@
         "height": 232.15942469988397,
         "width": 323.21100909416833
       },
-			"id": "74ab84c6-4e01-4d02-bdbc-e0b1a8310eb2",
+      "id": "8dbf7c42-1cbc-4b13-ae60-031f4d690cbd",
       "name": "Sticky Note3",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [840, -347.6337155918741]
+      "position": [
+        2160,
+        1452.366284408126
+      ]
     },
     {
       "parameters": {
@@ -350,11 +383,14 @@
           }
         }
       },
-			"id": "be7064ff-518f-4566-bd58-e588d863172d",
+      "id": "ca3ce84d-b32b-4f94-a51c-634b36d2553d",
       "name": "Response Empty - Include Full Response",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 1220]
+      "position": [
+        2220,
+        3420
+      ]
     },
     {
       "parameters": {
@@ -382,11 +418,14 @@
           }
         }
       },
-			"id": "7102d906-c9b7-42c8-a95d-bf277cc88b51",
+      "id": "de6858d8-8677-4e02-9617-24fa13005278",
       "name": "Pagination Off",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 1440]
+      "position": [
+        2220,
+        3640
+      ]
     },
     {
       "parameters": {
@@ -394,11 +433,14 @@
         "height": 373,
         "width": 363
       },
-			"id": "34fd6eb1-c480-4336-825c-062849a0c9e4",
+      "id": "dfee3f26-25e8-4db8-82d7-c3d93a565a35",
       "name": "Sticky Note4",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [840, 1400]
+      "position": [
+        2160,
+        3600
+      ]
     },
     {
       "parameters": {
@@ -421,11 +463,14 @@
           }
         }
       },
-			"id": "d17b4ef6-b603-4c2d-a313-98f4ee0bb610",
+      "id": "1ef00909-5412-49c8-9de4-920ebb29999c",
       "name": "Pagination Not Set",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 1620]
+      "position": [
+        2220,
+        3820
+      ]
     },
     {
       "parameters": {
@@ -433,11 +478,14 @@
         "height": 232.15942469988397,
         "width": 394.89100909416834
       },
-			"id": "f349bb03-7ee3-46c1-98e9-a88070e7f53f",
+      "id": "f7b8a43e-c176-4b35-8d9f-1ce027a9a6eb",
       "name": "Sticky Note5",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [840, 1840]
+      "position": [
+        2160,
+        4040
+      ]
     },
     {
       "parameters": {
@@ -452,11 +500,14 @@
         "include": "none",
         "options": {}
       },
-			"id": "3bf2681e-0c13-4998-bade-516faa1098a8",
+      "id": "046aa51f-ad74-4213-ad52-16b7f6bf0ad6",
       "name": "Edit Fields",
       "type": "n8n-nodes-base.set",
       "typeVersion": 3.2,
-			"position": [1080, 1920]
+      "position": [
+        2400,
+        4120
+      ]
     },
     {
       "parameters": {
@@ -490,11 +541,14 @@
           }
         }
       },
-			"id": "c82cbdd8-92c6-43c8-9e27-da9f6106035e",
+      "id": "a6eed8b6-0cd4-449f-a062-d2a6387fc327",
       "name": "Loop",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [900, 1920],
+      "position": [
+        2220,
+        4120
+      ],
       "continueOnFail": true
     },
     {
@@ -503,33 +557,42 @@
         "height": 458.3224664750446,
         "width": 323.21100909416833
       },
-			"id": "d0f3ef25-fdab-4d05-a913-9ac93d0c4b9c",
+      "id": "1ece8875-2a64-4a5c-9bb4-ea2aec1d3740",
       "name": "Sticky Note6",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [820, -1280]
+      "position": [
+        2140,
+        520
+      ]
     },
     {
       "parameters": {
         "content": "# Response Format: Autodetect\n",
         "width": 545.8929725020898
       },
-			"id": "f4c57fc9-8cc4-44ee-a4e6-73b2fa24f4d7",
+      "id": "96087f8a-f670-439f-a84d-24139fb82425",
       "name": "Sticky Note7",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [200, -1272]
+      "position": [
+        1520,
+        528
+      ]
     },
     {
       "parameters": {
         "content": "# Response Format: set",
         "width": 545.8929725020898
       },
-			"id": "368400f3-3e43-4f0c-9d4b-b5e01e7f91a5",
+      "id": "8c2c8849-bf67-480e-9d60-333d35b731af",
       "name": "Sticky Note8",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [220, -340]
+      "position": [
+        1540,
+        1460
+      ]
     },
     {
       "parameters": {
@@ -557,11 +620,14 @@
           }
         }
       },
-			"id": "e4bf07be-251f-4f65-88f8-50a247530ca1",
+      "id": "dffe62af-48de-4f81-9dff-9449b8eaed47",
       "name": "Complete Expression - JSON Autodetect set",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [880, -1192]
+      "position": [
+        2200,
+        608
+      ]
     },
     {
       "parameters": {
@@ -586,35 +652,47 @@
           }
         }
       },
-			"id": "b8f606c8-c559-4616-8760-b110b6dbecda",
+      "id": "029a965f-ffbb-463e-805a-9d799c1652ba",
       "name": "Complete Expression - JSON unset",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [880, -1012]
+      "position": [
+        2200,
+        788
+      ]
     },
     {
       "parameters": {},
-			"id": "bd5ecfef-05d2-4b09-906d-9989f162ecfa",
+      "id": "1d695252-948a-40de-bb8c-52c0a05e024a",
       "name": "No Operation, do nothing1",
       "type": "n8n-nodes-base.noOp",
       "typeVersion": 1,
-			"position": [420, 520]
+      "position": [
+        1740,
+        2320
+      ]
     },
     {
       "parameters": {},
-			"id": "2be55c5f-3986-4631-bb9e-54a1f87e95b9",
+      "id": "91eaf5ae-87dd-4eac-b144-e165ff55569d",
       "name": "Data 2",
       "type": "n8n-nodes-base.manualTrigger",
       "typeVersion": 1,
-			"position": [120, 0]
+      "position": [
+        1440,
+        1800
+      ]
     },
     {
       "parameters": {},
-			"id": "0e5cdaf4-5766-4c0c-b89b-790951caa9b2",
+      "id": "4279a32b-787d-4360-aee6-bbb95c3b0afa",
       "name": "Data 1",
       "type": "n8n-nodes-base.noOp",
       "typeVersion": 1,
-			"position": [440, -1020]
+      "position": [
+        1760,
+        780
+      ]
     },
     {
       "parameters": {
@@ -653,11 +731,14 @@
           }
         }
       },
-			"id": "d348ab0a-99c8-4151-bd15-fad4a247834e",
+      "id": "10a5e5bd-b0e0-4490-bb7f-0de29f717b77",
       "name": "Response Empty - Text1",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [880, -720]
+      "position": [
+        2200,
+        1080
+      ]
     },
     {
       "parameters": {
@@ -665,11 +746,14 @@
         "height": 437.60980047313967,
         "width": 323.31395441111135
       },
-			"id": "4be6d042-ee89-45da-bcac-07f9c4ce4dac",
+      "id": "45e77316-f3cb-4515-a777-f921e10edce6",
       "name": "Sticky Note9",
       "type": "n8n-nodes-base.stickyNote",
       "typeVersion": 1,
-			"position": [820, -806.2261914090556]
+      "position": [
+        2140,
+        993.7738085909444
+      ]
     },
     {
       "parameters": {
@@ -709,11 +793,113 @@
           }
         }
       },
-			"id": "5b482293-5176-4408-a56f-036a2a8b16e8",
+      "id": "7f0491f2-cd1d-4dc8-9cb7-26fce638c9b8",
       "name": "Response Empty - Include Full Response1",
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.1,
-			"position": [880, -540]
+      "position": [
+        2200,
+        1260
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "https://dummyjson.com/users",
+        "sendBody": true,
+        "contentType": "multipart-form-data",
+        "bodyParameters": {
+          "parameters": [
+            {
+              "name": "limit",
+              "value": "3"
+            },
+            {
+              "name": "skip",
+              "value": "0"
+            }
+          ]
+        },
+        "options": {
+          "response": {
+            "response": {
+              "responseFormat": "json"
+            }
+          },
+          "pagination": {
+            "pagination": {
+              "parameters": {
+                "parameters": [
+                  {
+                    "type": "body",
+                    "name": "skip",
+                    "value": "={{ $pageCount * 3 }}"
+                  }
+                ]
+              },
+              "limitPagesFetched": true,
+              "maxRequests": 3
+            }
+          }
+        }
+      },
+      "id": "2d4217e3-4c0c-48d1-87e9-fd46b370be4e",
+      "name": "POST Form Data",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.1,
+      "position": [
+        2220,
+        2680
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "https://dummyjson.com/users",
+        "sendBody": true,
+        "bodyParameters": {
+          "parameters": [
+            {
+              "name": "limit",
+              "value": "3"
+            },
+            {
+              "name": "skip",
+              "value": "0"
+            }
+          ]
+        },
+        "options": {
+          "response": {
+            "response": {
+              "responseFormat": "json"
+            }
+          },
+          "pagination": {
+            "pagination": {
+              "parameters": {
+                "parameters": [
+                  {
+                    "type": "body",
+                    "name": "skip",
+                    "value": "={{ $pageCount * 3 }}"
+                  }
+                ]
+              },
+              "limitPagesFetched": true,
+              "maxRequests": 3
+            }
+          }
+        }
+      },
+      "id": "a428f414-d13e-40d5-b1cf-beaf53df5884",
+      "name": "POST JSON",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.1,
+      "position": [
+        2220,
+        2500
+      ]
     }
   ],
   "pinData": {
@@ -1406,6 +1592,100 @@
           "statusMessage": null
         }
       }
+    ],
+    "POST Form Data": [
+      {
+        "json": {
+          "id": 0
+        }
+      },
+      {
+        "json": {
+          "id": 1
+        }
+      },
+      {
+        "json": {
+          "id": 2
+        }
+      },
+      {
+        "json": {
+          "id": 3
+        }
+      },
+      {
+        "json": {
+          "id": 4
+        }
+      },
+      {
+        "json": {
+          "id": 5
+        }
+      },
+      {
+        "json": {
+          "id": 6
+        }
+      },
+      {
+        "json": {
+          "id": 7
+        }
+      },
+      {
+        "json": {
+          "id": 8
+        }
+      }
+    ],
+    "POST JSON": [
+      {
+        "json": {
+          "id": 0
+        }
+      },
+      {
+        "json": {
+          "id": 1
+        }
+      },
+      {
+        "json": {
+          "id": 2
+        }
+      },
+      {
+        "json": {
+          "id": 3
+        }
+      },
+      {
+        "json": {
+          "id": 4
+        }
+      },
+      {
+        "json": {
+          "id": 5
+        }
+      },
+      {
+        "json": {
+          "id": 6
+        }
+      },
+      {
+        "json": {
+          "id": 7
+        }
+      },
+      {
+        "json": {
+          "id": 8
+        }
+      }
     ]
   },
   "connections": {
@@ -1477,6 +1757,16 @@
             "node": "Complete Expression - JSON",
             "type": "main",
             "index": 0
+          },
+          {
+            "node": "POST JSON",
+            "type": "main",
+            "index": 0
+          },
+          {
+            "node": "POST Form Data",
+            "type": "main",
+            "index": 0
           }
         ]
       ]
@@ -1528,10 +1818,11 @@
   "settings": {
     "executionOrder": "v1"
   },
-	"versionId": "24c0feef-1565-4936-a28f-7111760e6204",
+  "versionId": "107ed80e-d232-412e-ad52-9e902b46b2a4",
-	"id": "ZKVF9JrrzoxXbUqu",
   "meta": {
-		"instanceId": "021d3c82ba2d3bc090cbf4fc81c9312668bcc34297e022bb3438c5c88a43a5ff"
+    "templateCredsSetupCompleted": true,
+    "instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
   },
+  "id": "PZHNXWq2FzZp0F9A",
   "tags": []
 }