feat(Google Cloud Storage Node): Use streaming for file uploads (#6462)

fix(Google Cloud Storage Node): Use streaming for file uploads

packages/core/src/NodeExecuteFunctions.ts

@@ -732,7 +732,7 @@ function convertN8nRequestToAxios(n8nRequest: IHttpRequestOptions): AxiosRequest
 	// Destructure properties with the same name first.
 	const { headers, method, timeout, auth, proxy, url } = n8nRequest;
 
-	const axiosRequest = {
+	const axiosRequest: AxiosRequestConfig = {
 		headers: headers ?? {},
 		method,
 		timeout,

packages/nodes-base/nodes/Google/CloudStorage/ObjectDescription.ts

@@ -1,5 +1,11 @@
 import FormData from 'form-data';
-import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow';
+import type { Readable } from 'stream';
+import {
+	BINARY_ENCODING,
+	type IDataObject,
+	type INodeExecutionData,
+	type INodeProperties,
+} from 'n8n-workflow';
 
 // Define these because we'll be using them in two separate places
 const metagenerationFilters: INodeProperties[] = [
@@ -144,23 +150,31 @@ export const objectOperations: INodeProperties[] = [
 								});
 
 								// Determine content and content type
-								let content: string | Buffer;
+								let content: string | Buffer | Readable;
 								let contentType: string;
+								let contentLength: number;
 								if (useBinary) {
 									const binaryPropertyName = this.getNodeParameter(
 										'createBinaryPropertyName',
 									) as string;
 
 									const binaryData = this.helpers.assertBinaryData(binaryPropertyName);
-
-									// Decode from base64 for upload
-									content = Buffer.from(binaryData.data, 'base64');
-									contentType = binaryData.mimeType;
+									if (binaryData.id) {
+										content = this.helpers.getBinaryStream(binaryData.id);
+										const binaryMetadata = await this.helpers.getBinaryMetadata(binaryData.id);
+										contentType = binaryMetadata.mimeType ?? 'application/octet-stream';
+										contentLength = binaryMetadata.fileSize;
+									} else {
+										content = Buffer.from(binaryData.data, BINARY_ENCODING);
+										contentType = binaryData.mimeType;
+										contentLength = content.length;
+									}
 								} else {
 									content = this.getNodeParameter('createContent') as string;
 									contentType = 'text/plain';
+									contentLength = content.length;
 								}
-								body.append('file', content, { contentType });
+								body.append('file', content, { contentType, knownLength: contentLength });
 
 								// Set the headers
 								if (!requestOptions.headers) requestOptions.headers = {};
@@ -170,7 +184,7 @@ export const objectOperations: INodeProperties[] = [
 								] = `multipart/related; boundary=${body.getBoundary()}`;
 
 								// Return the request data
-								requestOptions.body = body.getBuffer();
+								requestOptions.body = body;
 								return requestOptions;
 							},
 						],

packages/workflow/package.json

@@ -53,6 +53,7 @@
     "crypto-js": "^4.1.1",
     "deep-equal": "^2.2.0",
     "esprima-next": "5.8.4",
+    "form-data": "^4.0.0",
     "jmespath": "^0.16.0",
     "js-base64": "^3.7.2",
     "lodash": "^4.17.21",

packages/workflow/src/Interfaces.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 // eslint-disable-next-line max-classes-per-file
 import type * as express from 'express';
-import type * as FormData from 'form-data';
+import type FormData from 'form-data';
 import type { IncomingHttpHeaders } from 'http';
 import type { Readable } from 'stream';
 import type { URLSearchParams } from 'url';

packages/workflow/src/utils.ts

@@ -1,3 +1,4 @@
+import FormData from 'form-data';
 import type { BinaryFileType, JsonObject } from './Interfaces';
 
 const readStreamClasses = new Set(['ReadStream', 'Readable', 'ReadableStream']);
@@ -5,6 +6,7 @@ const readStreamClasses = new Set(['ReadStream', 'Readable', 'ReadableStream']);
 export const isObjectEmpty = (obj: object | null | undefined): boolean => {
 	if (obj === undefined || obj === null) return true;
 	if (typeof obj === 'object') {
+		if (obj instanceof FormData) return obj.getLengthSync() === 0;
 		if (Array.isArray(obj)) return obj.length === 0;
 		if (obj instanceof Set || obj instanceof Map) return obj.size === 0;
 		if (ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer) return obj.byteLength === 0;

pnpm-lock.yaml

@@ -1297,6 +1297,9 @@ importers:
       esprima-next:
         specifier: 5.8.4
         version: 5.8.4
+      form-data:
+        specifier: ^4.0.0
+        version: 4.0.0
       jmespath:
         specifier: ^0.16.0
         version: 0.16.0