refactor(core): Reduce memory usage in the Webhook node (#4640)

use file streaming to pass webhook binaries around

packages/cli/src/Server.ts

@@ -1497,18 +1497,22 @@ class App {
 		// Binary data
 		// ----------------------------------------
 
-		// Returns binary buffer
+		// Download binary
 		this.app.get(
 			`/${this.restEndpoint}/data/:path`,
-			ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string> => {
+			async (req: express.Request, res: express.Response): Promise<void> => {
 				// TODO UM: check if this needs permission check for UM
-				const dataPath = req.params.path;
-				return BinaryDataManager.getInstance()
-					.retrieveBinaryDataByIdentifier(dataPath)
-					.then((buffer: Buffer) => {
-						return buffer.toString('base64');
-					});
-			}),
+				const identifier = req.params.path;
+				const binaryDataManager = BinaryDataManager.getInstance();
+				const binaryPath = binaryDataManager.getBinaryPath(identifier);
+				const { mimeType, fileName, fileSize } = await binaryDataManager.getBinaryMetadata(
+					identifier,
+				);
+				if (mimeType) res.setHeader('Content-Type', mimeType);
+				if (fileName) res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
+				res.setHeader('Content-Length', fileSize);
+				res.sendFile(binaryPath);
+			},
 		);
 
 		// ----------------------------------------

packages/core/package.json

@@ -56,6 +56,7 @@
     "n8n-workflow": "~0.126.0",
     "oauth-1.0a": "^2.2.6",
     "p-cancelable": "^2.0.0",
+    "pretty-bytes": "^5.6.0",
     "qs": "^6.10.1",
     "request": "^2.88.2",
     "request-promise-native": "^1.0.7",

packages/core/src/BinaryDataManager/FileSystem.ts

@@ -1,8 +1,9 @@
-import { promises as fs } from 'fs';
+import fs from 'fs/promises';
+import { jsonParse } from 'n8n-workflow';
 import path from 'path';
 import { v4 as uuid } from 'uuid';
 
-import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
 
 const PREFIX_METAFILE = 'binarymeta';
 const PREFIX_PERSISTED_METAFILE = 'persistedmeta';
@@ -43,17 +44,47 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 			.then(() => {});
 	}
 
+	async getFileSize(identifier: string): Promise<number> {
+		const stats = await fs.stat(this.getBinaryPath(identifier));
+		return stats.size;
+	}
+
+	async copyBinaryFile(filePath: string, executionId: string): Promise<string> {
+		const binaryDataId = this.generateFileName(executionId);
+		await this.addBinaryIdToPersistMeta(executionId, binaryDataId);
+		await this.copyFileToLocalStorage(filePath, binaryDataId);
+		return binaryDataId;
+	}
+
+	async storeBinaryMetadata(identifier: string, metadata: BinaryMetadata) {
+		await fs.writeFile(this.getMetadataPath(identifier), JSON.stringify(metadata), {
+			encoding: 'utf-8',
+		});
+	}
+
+	async getBinaryMetadata(identifier: string): Promise<BinaryMetadata> {
+		return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' }));
+	}
+
 	async storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string> {
 		const binaryDataId = this.generateFileName(executionId);
-		return this.addBinaryIdToPersistMeta(executionId, binaryDataId).then(async () =>
-			this.saveToLocalStorage(binaryBuffer, binaryDataId).then(() => binaryDataId),
-		);
+		await this.addBinaryIdToPersistMeta(executionId, binaryDataId);
+		await this.saveToLocalStorage(binaryBuffer, binaryDataId);
+		return binaryDataId;
 	}
 
 	async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
 		return this.retrieveFromLocalStorage(identifier);
 	}
 
+	getBinaryPath(identifier: string): string {
+		return path.join(this.storagePath, identifier);
+	}
+
+	getMetadataPath(identifier: string): string {
+		return path.join(this.storagePath, `${identifier}.metadata`);
+	}
+
 	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
 		const tt = new Date(new Date().getTime() + this.binaryDataTTL * 60000);
 		return fs.writeFile(
@@ -180,7 +211,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 	}
 
 	private generateFileName(prefix: string): string {
-		return `${prefix}_${uuid()}`;
+		return [prefix, uuid()].join('');
 	}
 
 	private getBinaryDataMetaPath() {
@@ -196,15 +227,19 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 	}
 
 	private async deleteFromLocalStorage(identifier: string) {
-		return fs.rm(path.join(this.storagePath, identifier));
+		return fs.rm(this.getBinaryPath(identifier));
+	}
+
+	private async copyFileToLocalStorage(source: string, identifier: string): Promise<void> {
+		await fs.cp(source, this.getBinaryPath(identifier));
 	}
 
 	private async saveToLocalStorage(data: Buffer, identifier: string) {
-		await fs.writeFile(path.join(this.storagePath, identifier), data);
+		await fs.writeFile(this.getBinaryPath(identifier), data);
 	}
 
 	private async retrieveFromLocalStorage(identifier: string): Promise<Buffer> {
-		const filePath = path.join(this.storagePath, identifier);
+		const filePath = this.getBinaryPath(identifier);
 		try {
 			return await fs.readFile(filePath);
 		} catch (e) {

packages/core/src/BinaryDataManager/index.ts

@@ -1,7 +1,9 @@
-import { IBinaryData, INodeExecutionData } from 'n8n-workflow';
+import prettyBytes from 'pretty-bytes';
+import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
 import { BINARY_ENCODING } from '../Constants';
-import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import type { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
 import { BinaryDataFileSystem } from './FileSystem';
+import { readFile, stat } from 'fs/promises';
 
 export class BinaryDataManager {
 	static instance: BinaryDataManager | undefined;
@@ -43,31 +45,59 @@ export class BinaryDataManager {
 		return BinaryDataManager.instance;
 	}
 
+	async copyBinaryFile(
+		binaryData: IBinaryData,
+		filePath: string,
+		executionId: string,
+	): Promise<IBinaryData> {
+		// If a manager handles this binary, copy over the binary file and return its reference id.
+		const manager = this.managers[this.binaryDataMode];
+		if (manager) {
+			const identifier = await manager.copyBinaryFile(filePath, executionId);
+			// Add data manager reference id.
+			binaryData.id = this.generateBinaryId(identifier);
+
+			// Prevent preserving data in memory if handled by a data manager.
+			binaryData.data = this.binaryDataMode;
+
+			const fileSize = await manager.getFileSize(identifier);
+			binaryData.fileSize = prettyBytes(fileSize);
+
+			await manager.storeBinaryMetadata(identifier, {
+				fileName: binaryData.fileName,
+				mimeType: binaryData.mimeType,
+				fileSize,
+			});
+		} else {
+			const { size } = await stat(filePath);
+			binaryData.fileSize = prettyBytes(size);
+			binaryData.data = await readFile(filePath, { encoding: BINARY_ENCODING });
+		}
+
+		return binaryData;
+	}
+
 	async storeBinaryData(
 		binaryData: IBinaryData,
 		binaryBuffer: Buffer,
 		executionId: string,
 	): Promise<IBinaryData> {
-		const retBinaryData = binaryData;
+		binaryData.fileSize = prettyBytes(binaryBuffer.length);
 
-		// If a manager handles this binary, return the binary data with it's reference id.
-		if (this.managers[this.binaryDataMode]) {
-			return this.managers[this.binaryDataMode]
-				.storeBinaryData(binaryBuffer, executionId)
-				.then((filename) => {
-					// Add data manager reference id.
-					retBinaryData.id = this.generateBinaryId(filename);
+		// If a manager handles this binary, return the binary data with its reference id.
+		const manager = this.managers[this.binaryDataMode];
+		if (manager) {
+			const identifier = await manager.storeBinaryData(binaryBuffer, executionId);
+			// Add data manager reference id.
+			binaryData.id = this.generateBinaryId(identifier);
 
-					// Prevent preserving data in memory if handled by a data manager.
-					retBinaryData.data = this.binaryDataMode;
-
-					// Short-circuit return to prevent further actions.
-					return retBinaryData;
-				});
+			// Prevent preserving data in memory if handled by a data manager.
+			binaryData.data = this.binaryDataMode;
+		} else {
+			// Else fallback to storing this data in memory.
+			binaryData.data = binaryBuffer.toString(BINARY_ENCODING);
 		}
 
-		// Else fallback to storing this data in memory.
-		retBinaryData.data = binaryBuffer.toString(BINARY_ENCODING);
 		return binaryData;
 	}
 
@@ -88,6 +118,24 @@ export class BinaryDataManager {
 		throw new Error('Storage mode used to store binary data not available');
 	}
 
+	getBinaryPath(identifier: string): string {
+		const { mode, id } = this.splitBinaryModeFileId(identifier);
+		if (this.managers[mode]) {
+			return this.managers[mode].getBinaryPath(id);
+		}
+
+		throw new Error('Storage mode used to store binary data not available');
+	}
+
+	async getBinaryMetadata(identifier: string): Promise<BinaryMetadata> {
+		const { mode, id } = this.splitBinaryModeFileId(identifier);
+		if (this.managers[mode]) {
+			return this.managers[mode].getBinaryMetadata(id);
+		}
+
+		throw new Error('Storage mode used to store binary data not available');
+	}
+
 	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
 		if (this.managers[this.binaryDataMode]) {
 			return this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(executionId);

packages/core/src/Interfaces.ts

@@ -260,6 +260,7 @@ export interface IWebhookFunctions extends IWebhookFunctionsBase {
 			filePath?: string,
 			mimeType?: string,
 		): Promise<IBinaryData>;
+		copyBinaryFile(filePath: string, fileName: string, mimeType?: string): Promise<IBinaryData>;
 		request: (uriOrObject: string | IDataObject | any, options?: IDataObject) => Promise<any>;
 		requestWithAuthentication(
 			this: IAllExecuteFunctions,
@@ -306,10 +307,21 @@ export interface IBinaryDataConfig {
 	persistedBinaryDataTTL: number;
 }
 
+export interface BinaryMetadata {
+	fileName?: string;
+	mimeType?: string;
+	fileSize: number;
+}
+
 export interface IBinaryDataManager {
 	init(startPurger: boolean): Promise<void>;
+	getFileSize(filePath: string): Promise<number>;
+	copyBinaryFile(filePath: string, executionId: string): Promise<string>;
+	storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise<void>;
+	getBinaryMetadata(identifier: string): Promise<BinaryMetadata>;
 	storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string>;
 	retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer>;
+	getBinaryPath(identifier: string): string;
 	markDataForDeletionByExecutionId(executionId: string): Promise<void>;
 	deleteMarkedFiles(): Promise<unknown>;
 	deleteBinaryDataByIdentifier(identifier: string): Promise<void>;

packages/core/src/NodeExecuteFunctions.ts

@@ -64,6 +64,7 @@ import {
 	NodeExecutionWithMetadata,
 	IPairedItemData,
 	deepCopy,
+	BinaryFileType,
 } from 'n8n-workflow';
 
 import { Agent } from 'https';
@@ -77,8 +78,8 @@ import FormData from 'form-data';
 import path from 'path';
 import { OptionsWithUri, OptionsWithUrl, RequestCallback, RequiredUriUrl } from 'request';
 import requestPromise, { RequestPromiseOptions } from 'request-promise-native';
-import { fromBuffer } from 'file-type';
-import { lookup } from 'mime-types';
+import FileType from 'file-type';
+import { lookup, extension } from 'mime-types';
 import { IncomingHttpHeaders } from 'http';
 import axios, {
 	AxiosError,
@@ -830,6 +831,13 @@ export async function getBinaryDataBuffer(
 	return BinaryDataManager.getInstance().retrieveBinaryData(binaryData);
 }
 
+function fileTypeFromMimeType(mimeType: string): BinaryFileType | undefined {
+	if (mimeType.startsWith('image/')) return 'image';
+	if (mimeType.startsWith('video/')) return 'video';
+	if (mimeType.startsWith('text/') || mimeType.startsWith('application/json')) return 'text';
+	return;
+}
+
 /**
  * Store an incoming IBinaryData & related buffer using the configured binary data manager.
  *
@@ -846,10 +854,60 @@ export async function setBinaryDataBuffer(
 	return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId);
 }
 
+export async function copyBinaryFile(
+	executionId: string,
+	filePath: string,
+	fileName: string,
+	mimeType?: string,
+): Promise<IBinaryData> {
+	let fileExtension: string | undefined;
+	if (!mimeType) {
+		// If no mime type is given figure it out
+
+		if (filePath) {
+			// Use file path to guess mime type
+			const mimeTypeLookup = lookup(filePath);
+			if (mimeTypeLookup) {
+				mimeType = mimeTypeLookup;
+			}
+		}
+
+		if (!mimeType) {
+			// read the first bytes of the file to guess mime type
+			const fileTypeData = await FileType.fromFile(filePath);
+			if (fileTypeData) {
+				mimeType = fileTypeData.mime;
+				fileExtension = fileTypeData.ext;
+			}
+		}
+
+		if (!mimeType) {
+			// Fall back to text
+			mimeType = 'text/plain';
+		}
+	} else if (!fileExtension) {
+		fileExtension = extension(mimeType) || undefined;
+	}
+
+	const returnData: IBinaryData = {
+		mimeType,
+		fileType: fileTypeFromMimeType(mimeType),
+		fileExtension,
+		data: '',
+	};
+
+	if (fileName) {
+		returnData.fileName = fileName;
+	} else if (filePath) {
+		returnData.fileName = path.parse(filePath).base;
+	}
+
+	return BinaryDataManager.getInstance().copyBinaryFile(returnData, filePath, executionId);
+}
+
 /**
  * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as
  * base64 and adds metadata.
- *
  */
 export async function prepareBinaryData(
 	binaryData: Buffer,
@@ -871,7 +929,7 @@ export async function prepareBinaryData(
 
 		if (!mimeType) {
 			// Use buffer to guess mime type
-			const fileTypeData = await fromBuffer(binaryData);
+			const fileTypeData = await FileType.fromBuffer(binaryData);
 			if (fileTypeData) {
 				mimeType = fileTypeData.mime;
 				fileExtension = fileTypeData.ext;
@@ -882,10 +940,13 @@ export async function prepareBinaryData(
 			// Fall back to text
 			mimeType = 'text/plain';
 		}
+	} else if (!fileExtension) {
+		fileExtension = extension(mimeType) || undefined;
 	}
 
 	const returnData: IBinaryData = {
 		mimeType,
+		fileType: fileTypeFromMimeType(mimeType),
 		fileExtension,
 		data: '',
 	};
@@ -3076,6 +3137,19 @@ export function getExecuteWebhookFunctions(
 				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
 					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
 				},
+				async copyBinaryFile(
+					filePath: string,
+					fileName: string,
+					mimeType?: string,
+				): Promise<IBinaryData> {
+					return copyBinaryFile.call(
+						this,
+						additionalData.executionId!,
+						filePath,
+						fileName,
+						mimeType,
+					);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,

packages/editor-ui/src/Interface.ts

@@ -215,6 +215,7 @@ export interface IRestApi {
 	retryExecution(id: string, loadWorkflow?: boolean): Promise<boolean>;
 	getTimezones(): Promise<IDataObject>;
 	getBinaryBufferString(dataPath: string): Promise<string>;
+	getBinaryUrl(dataPath: string): string;
 }
 
 export interface INodeTranslationHeaders {
@@ -226,14 +227,6 @@ export interface INodeTranslationHeaders {
 	};
 }
 
-export interface IBinaryDisplayData {
-	index: number;
-	key: string;
-	node: string;
-	outputIndex: number;
-	runIndex: number;
-}
-
 export interface IStartRunData {
 	workflowData: IWorkflowData;
 	startNodes?: string[];

packages/editor-ui/src/components/BinaryDataDisplay.vue

@@ -20,10 +20,7 @@
 </template>
 
 <script lang="ts">
-import {
-	IBinaryData,
-	IRunData,
-} from 'n8n-workflow';
+import type { IBinaryData, IRunData } from 'n8n-workflow';
 
 import BinaryDataDisplayEmbed from '@/components/BinaryDataDisplayEmbed.vue';
 
@@ -44,7 +41,7 @@ export default mixins(
 			BinaryDataDisplayEmbed,
 		},
 		props: [
-			'displayData', // IBinaryDisplayData
+			'displayData', // IBinaryData
 			'windowVisible', // boolean
 		],
 		computed: {
@@ -67,14 +64,6 @@ export default mixins(
 				return binaryDataItem;
 			},
 
-			embedClass (): string[] {
-				// @ts-ignore
-				if (this.binaryData! !== null && this.binaryData!.mimeType! !== undefined && (this.binaryData!.mimeType! as string).startsWith('image')) {
-					return ['image'];
-				}
-				return ['other'];
-			},
-
 			workflowRunData (): IRunData | null {
 				const workflowExecution = this.workflowsStore.getWorkflowExecution;
 				if (workflowExecution === null) {

packages/editor-ui/src/components/BinaryDataDisplayEmbed.vue

@@ -7,7 +7,7 @@
 			Error loading binary data
 		</div>
 		<span v-else>
-			<video v-if="binaryData.mimeType && binaryData.mimeType.startsWith('video/')" controls autoplay>
+			<video v-if="binaryData.fileType === 'video'" controls autoplay>
 				<source :src="embedSource" :type="binaryData.mimeType">
 				{{ $locale.baseText('binaryDataDisplay.yourBrowserDoesNotSupport') }}
 			</video>
@@ -17,10 +17,9 @@
 </template>
 
 <script lang="ts">
-
-
 import mixins from 'vue-typed-mixins';
 import { restApi } from '@/mixins/restApi';
+import type { IBinaryData } from 'n8n-workflow';
 
 export default mixins(
 	restApi,
@@ -28,7 +27,7 @@ export default mixins(
 	.extend({
 		name: 'BinaryDataDisplayEmbed',
 		props: [
-			'binaryData', // IBinaryDisplayData
+			'binaryData', // IBinaryData
 		],
 		data() {
 			return {
@@ -38,15 +37,15 @@ export default mixins(
 			};
 		},
 		async mounted() {
-			if(!this.binaryData.id) {
+			const id = this.binaryData?.id;
+			if(!id) {
 				this.embedSource = 'data:' + this.binaryData.mimeType + ';base64,' + this.binaryData.data;
 				this.isLoading = false;
 				return;
 			}
 
 			try {
-				const bufferString = await this.restApi().getBinaryBufferString(this.binaryData!.id!);
-				this.embedSource = 'data:' + this.binaryData.mimeType + ';base64,' + bufferString;
+				this.embedSource = this.restApi().getBinaryUrl(id);
 				this.isLoading = false;
 			} catch (e) {
 				this.isLoading = false;
@@ -55,11 +54,8 @@ export default mixins(
 		},
 		methods: {
 			embedClass(): string[] {
-				// @ts-ignore
-				if (this.binaryData! !== null && this.binaryData!.mimeType! !== undefined && (this.binaryData!.mimeType! as string).startsWith('image')) {
-					return ['image'];
-				}
-				return ['other'];
+				const { fileType } = (this.binaryData || {}) as IBinaryData;
+				return [fileType ?? 'other'];
 			},
 		},
 	});

packages/editor-ui/src/components/RunData.vue

@@ -282,9 +282,13 @@
 									<div><n8n-text size="small" :bold="true">{{ $locale.baseText('runData.mimeType') }}: </n8n-text></div>
 									<div :class="$style.binaryValue">{{binaryData.mimeType}}</div>
 								</div>
+								<div v-if="binaryData.fileSize">
+									<div><n8n-text size="small" :bold="true">{{ $locale.baseText('runData.fileSize') }}: </n8n-text></div>
+									<div :class="$style.binaryValue">{{binaryData.fileSize}}</div>
+								</div>
 
 								<div :class="$style.binaryButtonContainer">
-									<n8n-button size="small" :label="$locale.baseText('runData.showBinaryData')" class="binary-data-show-data-button" @click="displayBinaryData(index, key)" />
+									<n8n-button v-if="isViewable(index, key)" size="small" :label="$locale.baseText('runData.showBinaryData')" class="binary-data-show-data-button" @click="displayBinaryData(index, key)" />
 									<n8n-button v-if="isDownloadable(index, key)" size="small" type="secondary" :label="$locale.baseText('runData.downloadBinaryData')" class="binary-data-show-data-button" @click="downloadBinaryData(index, key)" />
 								</div>
 							</div>
@@ -341,7 +345,6 @@ import {
 } from 'n8n-workflow';
 
 import {
-	IBinaryDisplayData,
 	IExecutionResponse,
 	INodeUi,
 	INodeUpdatePropertiesInformation,
@@ -363,7 +366,6 @@ import BinaryDataDisplay from '@/components/BinaryDataDisplay.vue';
 import WarningTooltip from '@/components/WarningTooltip.vue';
 import NodeErrorView from '@/components/Error/NodeErrorView.vue';
 
-import { copyPaste } from '@/mixins/copyPaste';
 import { externalHooks } from "@/mixins/externalHooks";
 import { genericHelpers } from '@/mixins/genericHelpers';
 import { nodeHelpers } from '@/mixins/nodeHelpers';
@@ -385,7 +387,6 @@ export type EnterEditModeArgs = {
 };
 
 export default mixins(
-	copyPaste,
 	externalHooks,
 	genericHelpers,
 	nodeHelpers,
@@ -460,7 +461,7 @@ export default mixins(
 				showData: false,
 				outputIndex: 0,
 				binaryDataDisplayVisible: false,
-				binaryDataDisplayData: null as IBinaryDisplayData | null,
+				binaryDataDisplayData: null as IBinaryData | null,
 
 				MAX_DISPLAY_DATA_SIZE,
 				MAX_DISPLAY_ITEMS_AUTO_ALL,
@@ -1041,23 +1042,26 @@ export default mixins(
 				this.workflowsStore.setWorkflowExecutionData(null);
 				this.updateNodesExecutionIssues();
 			},
+			isViewable (index: number, key: string): boolean {
+				const { fileType }: IBinaryData = this.binaryData[index][key];
+				return !!fileType && ['image', 'video'].includes(fileType);
+			},
 			isDownloadable (index: number, key: string): boolean {
-				const binaryDataItem: IBinaryData = this.binaryData[index][key];
-				return !!(binaryDataItem.mimeType && binaryDataItem.fileName);
+				const { mimeType, fileName }: IBinaryData = this.binaryData[index][key];
+				return !!(mimeType && fileName);
 			},
 			async downloadBinaryData (index: number, key: string) {
-				const binaryDataItem: IBinaryData = this.binaryData[index][key];
+				const { id, data, fileName, fileExtension, mimeType }: IBinaryData = this.binaryData[index][key];
 
-				let bufferString = 'data:' + binaryDataItem.mimeType + ';base64,';
-				if(binaryDataItem.id) {
-					bufferString += await this.restApi().getBinaryBufferString(binaryDataItem.id);
+				if(id) {
+					const url = this.restApi().getBinaryUrl(id);
+					saveAs(url, [fileName, fileExtension].join('.'));
+					return;
 				} else {
-					bufferString += binaryDataItem.data;
+					const bufferString = 'data:' + mimeType + ';base64,' + data;
+					const blob = await fetch(bufferString).then(d => d.blob());
+					saveAs(blob, fileName);
 				}
-
-				const data = await fetch(bufferString);
-				const blob = await data.blob();
-				saveAs(blob, binaryDataItem.fileName);
 			},
 			displayBinaryData (index: number, key: string) {
 				this.binaryDataDisplayVisible = true;

packages/editor-ui/src/mixins/restApi.ts

@@ -182,6 +182,10 @@ export const restApi = Vue.extend({
 				getBinaryBufferString: (dataPath: string): Promise<string> => {
 					return self.restApi().makeRestApiRequest('GET', `/data/${dataPath}`);
 				},
+
+				getBinaryUrl: (dataPath: string): string => {
+					return self.rootStore.getRestApiContext.baseUrl + `/data/${dataPath}`;
+				},
 			};
 		},
 	},

packages/editor-ui/src/plugins/i18n/locales/en.json

@@ -982,6 +982,7 @@
 	"runData.items": "Items",
 	"runData.json": "JSON",
 	"runData.mimeType": "Mime Type",
+	"runData.fileSize": "File Size",
 	"runData.ms": "ms",
 	"runData.noBinaryDataFound": "No binary data found",
 	"runData.noData": "No data",

packages/nodes-base/nodes/Wait/Wait.node.ts

@@ -11,15 +11,16 @@ import {
 	NodeOperationError,
 } from 'n8n-workflow';
 
-import basicAuth from 'basic-auth';
-
-import { Response } from 'express';
-
 import fs from 'fs';
-
+import stream from 'stream';
+import { promisify } from 'util';
+import basicAuth from 'basic-auth';
+import type { Response } from 'express';
 import formidable from 'formidable';
-
 import isbot from 'isbot';
+import { file as tmpFile } from 'tmp-promise';
+
+const pipeline = promisify(stream.pipeline);
 
 function authorizationError(resp: Response, realm: string, responseCode: number, message?: string) {
 	if (message === undefined) {
@@ -673,10 +674,8 @@ export class Wait implements INodeType {
 			}
 		}
 
-		// @ts-ignore
-		const mimeType = headers['content-type'] || 'application/json';
+		const mimeType = headers['content-type'] ?? 'application/json';
 		if (mimeType.includes('multipart/form-data')) {
-			// @ts-ignore
 			const form = new formidable.IncomingForm({ multiples: true });
 
 			return new Promise((resolve, _reject) => {
@@ -715,12 +714,10 @@ export class Wait implements INodeType {
 								binaryPropertyName = `${options.binaryPropertyName}${count}`;
 							}
 
-							const fileJson = file.toJSON() as unknown as IDataObject;
-							const fileContent = await fs.promises.readFile(file.path);
-
-							returnItem.binary![binaryPropertyName] = await this.helpers.prepareBinaryData(
-								Buffer.from(fileContent),
-								fileJson.name as string,
+							const fileJson = file.toJSON();
+							returnItem.binary![binaryPropertyName] = await this.helpers.copyBinaryFile(
+								file.path,
+								fileJson.name || fileJson.filename,
 								fileJson.type as string,
 							);
 
@@ -735,38 +732,35 @@ export class Wait implements INodeType {
 		}
 
 		if (options.binaryData === true) {
-			return new Promise((resolve, _reject) => {
-				const binaryPropertyName = options.binaryPropertyName || 'data';
-				const data: Buffer[] = [];
+			const binaryFile = await tmpFile({ prefix: 'n8n-webhook-' });
 
-				req.on('data', (chunk) => {
-					data.push(chunk);
-				});
+			try {
+				await pipeline(req, fs.createWriteStream(binaryFile.path));
 
-				req.on('end', async () => {
-					const returnItem: INodeExecutionData = {
-						binary: {},
-						json: {
-							headers,
-							params: this.getParamsData(),
-							query: this.getQueryData(),
-							body: this.getBodyData(),
-						},
-					};
+				const returnItem: INodeExecutionData = {
+					binary: {},
+					json: {
+						headers,
+						params: this.getParamsData(),
+						query: this.getQueryData(),
+						body: this.getBodyData(),
+					},
+				};
 
-					returnItem.binary![binaryPropertyName as string] = await this.helpers.prepareBinaryData(
-						Buffer.concat(data),
-					);
+				const binaryPropertyName = (options.binaryPropertyName || 'data') as string;
+				returnItem.binary![binaryPropertyName] = await this.helpers.copyBinaryFile(
+					binaryFile.path,
+					mimeType,
+				);
 
-					return resolve({
-						workflowData: [[returnItem]],
-					});
-				});
-
-				req.on('error', (error) => {
-					throw new NodeOperationError(this.getNode(), error);
-				});
-			});
+				return {
+					workflowData: [[returnItem]],
+				};
+			} catch (error) {
+				throw new NodeOperationError(this.getNode(), error);
+			} finally {
+				await binaryFile.cleanup();
+			}
 		}
 
 		const response: INodeExecutionData = {

packages/nodes-base/nodes/Webhook/Webhook.node.ts

@@ -10,15 +10,16 @@ import {
 	NodeOperationError,
 } from 'n8n-workflow';
 
-import basicAuth from 'basic-auth';
-
-import { Response } from 'express';
-
 import fs from 'fs';
-
+import stream from 'stream';
+import { promisify } from 'util';
+import basicAuth from 'basic-auth';
+import type { Response } from 'express';
 import formidable from 'formidable';
-
 import isbot from 'isbot';
+import { file as tmpFile } from 'tmp-promise';
+
+const pipeline = promisify(stream.pipeline);
 
 function authorizationError(resp: Response, realm: string, responseCode: number, message?: string) {
 	if (message === undefined) {
@@ -485,10 +486,8 @@ export class Webhook implements INodeType {
 			}
 		}
 
-		// @ts-ignore
-		const mimeType = headers['content-type'] || 'application/json';
+		const mimeType = headers['content-type'] ?? 'application/json';
 		if (mimeType.includes('multipart/form-data')) {
-			// @ts-ignore
 			const form = new formidable.IncomingForm({ multiples: true });
 
 			return new Promise((resolve, _reject) => {
@@ -527,12 +526,10 @@ export class Webhook implements INodeType {
 								binaryPropertyName = `${options.binaryPropertyName}${count}`;
 							}
 
-							const fileJson = file.toJSON() as unknown as IDataObject;
-							const fileContent = await fs.promises.readFile(file.path);
-
-							returnItem.binary![binaryPropertyName] = await this.helpers.prepareBinaryData(
-								Buffer.from(fileContent),
-								fileJson.name as string,
+							const fileJson = file.toJSON();
+							returnItem.binary![binaryPropertyName] = await this.helpers.copyBinaryFile(
+								file.path,
+								fileJson.name || fileJson.filename,
 								fileJson.type as string,
 							);
 
@@ -547,38 +544,35 @@ export class Webhook implements INodeType {
 		}
 
 		if (options.binaryData === true) {
-			return new Promise((resolve, _reject) => {
-				const binaryPropertyName = options.binaryPropertyName || 'data';
-				const data: Buffer[] = [];
+			const binaryFile = await tmpFile({ prefix: 'n8n-webhook-' });
 
-				req.on('data', (chunk) => {
-					data.push(chunk);
-				});
+			try {
+				await pipeline(req, fs.createWriteStream(binaryFile.path));
 
-				req.on('end', async () => {
-					const returnItem: INodeExecutionData = {
-						binary: {},
-						json: {
-							headers,
-							params: this.getParamsData(),
-							query: this.getQueryData(),
-							body: this.getBodyData(),
-						},
-					};
+				const returnItem: INodeExecutionData = {
+					binary: {},
+					json: {
+						headers,
+						params: this.getParamsData(),
+						query: this.getQueryData(),
+						body: this.getBodyData(),
+					},
+				};
 
-					returnItem.binary![binaryPropertyName as string] = await this.helpers.prepareBinaryData(
-						Buffer.concat(data),
-					);
+				const binaryPropertyName = (options.binaryPropertyName || 'data') as string;
+				returnItem.binary![binaryPropertyName] = await this.helpers.copyBinaryFile(
+					binaryFile.path,
+					mimeType,
+				);
 
-					return resolve({
-						workflowData: [[returnItem]],
-					});
-				});
-
-				req.on('error', (error) => {
-					throw new NodeOperationError(this.getNode(), error);
-				});
-			});
+				return {
+					workflowData: [[returnItem]],
+				};
+			} catch (error) {
+				throw new NodeOperationError(this.getNode(), error);
+			} finally {
+				await binaryFile.cleanup();
+			}
 		}
 
 		const response: INodeExecutionData = {

packages/workflow/src/Interfaces.ts

@@ -27,13 +27,16 @@ export type IAllExecuteFunctions =
 	| ITriggerFunctions
 	| IWebhookFunctions;
 
+export type BinaryFileType = 'text' | 'image' | 'video';
 export interface IBinaryData {
 	[key: string]: string | undefined;
 	data: string;
 	mimeType: string;
+	fileType?: BinaryFileType;
 	fileName?: string;
 	directory?: string;
 	fileExtension?: string;
+	fileSize?: string;
 	id?: string;
 }
 

pnpm-lock.yaml

@@ -349,6 +349,7 @@ importers:
       n8n-workflow: ~0.126.0
       oauth-1.0a: ^2.2.6
       p-cancelable: ^2.0.0
+      pretty-bytes: ^5.6.0
       qs: ^6.10.1
       request: ^2.88.2
       request-promise-native: ^1.0.7
@@ -367,6 +368,7 @@ importers:
       n8n-workflow: link:../workflow
       oauth-1.0a: 2.2.6
       p-cancelable: 2.1.1
+      pretty-bytes: 5.6.0
       qs: 6.11.0
       request: 2.88.2
       request-promise-native: 1.0.9_request@2.88.2
@@ -17465,7 +17467,6 @@ packages:
   /pretty-bytes/5.6.0:
     resolution: {integrity: sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==}
     engines: {node: '>=6'}
-    dev: true
 
   /pretty-error/2.1.2:
     resolution: {integrity: sha512-EY5oDzmsX5wvuynAByrmY0P0hcp+QpnAKbJng2A2MPjVKXCxrDSUkzghVJ4ZGPIv+JC4gX8fPUWscC0RtjsWGw==}