✨ Copy data on execution only if needed

packages/core/package.json

@@ -39,6 +39,7 @@
     "dependencies": {
         "crypto-js": "^3.1.9-1",
         "lodash.get": "^4.4.2",
+        "lodash.merge": "^4.6.2",
         "mmmagic": "^0.5.2",
         "n8n-workflow": "^0.7.0",
         "request-promise-native": "^1.0.7"

packages/core/src/WorkflowExecute.ts

@@ -20,6 +20,7 @@ import {
 	NodeExecuteFunctions,
 } from './';
 
+import { merge } from 'lodash';
 
 export class WorkflowExecute {
 	private additionalData: IWorkflowExecuteAdditionalData;
@@ -600,7 +601,7 @@ export class WorkflowExecute {
 						}
 
 						runExecutionData.resultData.lastNodeExecuted = executionData.node.name;
-						nodeSuccessData = await workflow.runNode(executionData.node, JSON.parse(JSON.stringify(executionData.data)), runExecutionData, runIndex, this.additionalData, NodeExecuteFunctions, this.mode);
+						nodeSuccessData = await workflow.runNode(executionData.node, executionData.data, runExecutionData, runIndex, this.additionalData, NodeExecuteFunctions, this.mode);
 
 						if (nodeSuccessData === null) {
 							// If null gets returned it means that the node did succeed
@@ -638,7 +639,7 @@ export class WorkflowExecute {
 							// Simply get the input data of the node if it has any and pass it through
 							// to the next node
 							if (executionData.data.main[0] !== null) {
-								nodeSuccessData = [(JSON.parse(JSON.stringify(executionData.data.main[0])) as INodeExecutionData[])];
+								nodeSuccessData = [executionData.data.main[0] as INodeExecutionData[]];
 							}
 						}
 					} else {

packages/nodes-base/nodes/Chargebee/Chargebee.node.ts

@@ -621,8 +621,11 @@ export class Chargebee implements INodeType {
 					returnData.push(data.invoice as IDataObject);
 				});
 			} else if (resource === 'invoice' && operation === 'pdfUrl') {
-				item.json.pdfUrl = responseData.download.download_url;
+				const data: IDataObject = {};
-				returnData.push(item.json);
+				Object.assign(data, items[i].json);
+
+				data.pdfUrl = responseData.download.download_url;
+				returnData.push(data);
 			} else {
 				returnData.push(responseData);
 			}

packages/nodes-base/nodes/Dropbox/Dropbox.node.ts

@@ -627,10 +627,21 @@ export class Dropbox implements INodeType {
 			}
 
 			if (resource === 'file' && operation === 'download') {
-				// TODO: Has to check if it already exists and only add if not
+
-				if (items[i].binary === undefined) {
+				const newItem: INodeExecutionData = {
-					items[i].binary = {};
+					json: items[i].json,
+					binary: {},
+				};
+
+				if (items[i].binary !== undefined) {
+					// Create a shallow copy of the binary data so that the old
+					// data references which do not get changed still stay behind
+					// but the incoming data does not get changed.
+					Object.assign(newItem.binary, items[i].binary);
 				}
+
+				items[i] = newItem;
+
 				const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string;
 
 				const filePathDownload = this.getNodeParameter('path', i) as string;

packages/nodes-base/nodes/EditImage.node.ts

@@ -538,6 +538,18 @@ export class EditImage implements INodeType {
 			throw new Error(`The operation "${operation}" is not supported!`);
 		}
 
+		const newItem: INodeExecutionData = {
+			json: item.json,
+			binary: {},
+		};
+
+		if (item.binary !== undefined) {
+			// Create a shallow copy of the binary data so that the old
+			// data references which do not get changed still stay behind
+			// but the incoming data does not get changed.
+			Object.assign(newItem.binary, item.binary);
+		}
+
 		return new Promise<INodeExecutionData>((resolve, reject) => {
 			gmInstance
 				.toBuffer((error: Error, buffer: Buffer) => {
@@ -545,9 +557,9 @@ export class EditImage implements INodeType {
 						return reject(error);
 					}
 
-					item.binary![dataPropertyName as string].data = buffer.toString(BINARY_ENCODING);
+					newItem.binary![dataPropertyName as string].data = buffer.toString(BINARY_ENCODING);
 
-					return resolve(item);
+					return resolve(newItem);
 				});
 		});
 	}

packages/nodes-base/nodes/Function.node.ts

@@ -40,6 +40,9 @@ export class Function implements INodeType {
 		// const item = this.getInputData();
 		let items = this.getInputData();
 
+		// Copy the items as they may get changed in the functions
+		items = JSON.parse(JSON.stringify(items));
+
 		// Define the global objects for the custom function
 		const sandbox = {
 			getNodeParameter: this.getNodeParameter,

packages/nodes-base/nodes/FunctionItem.node.ts

@@ -39,7 +39,10 @@ export class FunctionItem implements INodeType {
 	};
 
 	async executeSingle(this: IExecuteSingleFunctions): Promise<INodeExecutionData> {
-		const item = this.getInputData();
+		let item = this.getInputData();
+
+		// Copy the items as they may get changed in the functions
+		item = JSON.parse(JSON.stringify(item));
 
 		// Define the global objects for the custom function
 		const sandbox = {

packages/nodes-base/nodes/Github/Github.node.ts

@@ -1366,10 +1366,23 @@ export class Github implements INodeType {
 				if (asBinaryProperty === true) {
 					// Add the returned data to the item as binary property
 					const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string;
-					if (items[i].binary === undefined) {
+
-						items[i].binary = {};
+					const newItem: INodeExecutionData = {
+						json: items[i].json,
+						binary: {},
+					};
+
+					if (items[i].binary !== undefined) {
+						// Create a shallow copy of the binary data so that the old
+						// data references which do not get changed still stay behind
+						// but the incoming data does not get changed.
+						Object.assign(newItem.binary, items[i].binary);
 					}
-					items[i].binary![binaryPropertyName] = await this.helpers.prepareBinaryData(Buffer.from(responseData.content, 'base64'), responseData.path);
+
+					newItem.binary![binaryPropertyName] = await this.helpers.prepareBinaryData(Buffer.from(responseData.content, 'base64'), responseData.path);
+
+					items[i] = newItem;
+
 					return this.prepareOutputData(items);
 				}
 			}

packages/nodes-base/nodes/HttpRequest.node.ts

@@ -332,7 +332,6 @@ export class HttpRequest implements INodeType {
 		const httpBasicAuth = this.getCredentials('httpBasicAuth');
 		const httpHeaderAuth = this.getCredentials('httpHeaderAuth');
 
-		let item: INodeExecutionData;
 		let url: string, responseFormat: string;
 		let requestOptions: OptionsWithUri;
 		let setUiParameter: IDataObject;
@@ -360,8 +359,6 @@ export class HttpRequest implements INodeType {
 
 		const returnItems: INodeExecutionData[] = [];
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
-			item = items[itemIndex];
-
 			url = this.getNodeParameter('url', itemIndex) as string;
 			responseFormat = this.getNodeParameter('responseFormat', itemIndex) as string;
 

packages/nodes-base/nodes/LinkFish/LinkFish.node.ts

@@ -313,11 +313,20 @@ export class LinkFish implements INodeType {
 		if (operation === 'screenshot') {
 			const item = this.getInputData();
 
+			const newItem: INodeExecutionData = {
+				json: item.json,
+				binary: {},
+			};
+
+			if (item.binary !== undefined) {
+				// Create a shallow copy of the binary data so that the old
+				// data references which do not get changed still stay behind
+				// but the incoming data does not get changed.
+				Object.assign(newItem.binary, item.binary);
+			}
+
 			// Add the returned data to the item as binary property
 			const binaryPropertyName = this.getNodeParameter('binaryPropertyName') as string;
-			if (item.binary === undefined) {
-				item.binary = {};
-			}
 
 			let fileExtension = 'png';
 			let mimeType = 'image/png';
@@ -326,9 +335,9 @@ export class LinkFish implements INodeType {
 				mimeType = 'image/jpeg';
 			}
 
-			item.binary![binaryPropertyName] = await this.helpers.prepareBinaryData(responseData, `screenshot.${fileExtension}`, mimeType);
+			newItem.binary![binaryPropertyName] = await this.helpers.prepareBinaryData(responseData, `screenshot.${fileExtension}`, mimeType);
 
-			return item;
+			return newItem;
 		}
 
 		return {

packages/nodes-base/nodes/Merge.node.ts

@@ -178,14 +178,19 @@ export class Merge implements INodeType {
 					if (['null', 'undefined'].includes(typeof referenceValue)) {
 						continue;
 					}
+
+					// Copy the entry as the data gets changed
+					entry = JSON.parse(JSON.stringify(entry));
+
 					for (key of Object.keys(copyData[referenceValue as string].json)) {
 						// TODO: Currently only copies json data and no binary one
 						entry.json[key] = copyData[referenceValue as string].json[key];
 					}
 				}
+				returnData.push(entry);
 			}
 
-			return [dataInput1];
+			return [returnData];
 		} else if (mode === 'passThrough') {
 			const output = this.getNodeParameter('output', 0) as string;
 

packages/nodes-base/nodes/NextCloud/NextCloud.node.ts

@@ -443,7 +443,7 @@ export class NextCloud implements INodeType {
 
 
 	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
-		const items = this.getInputData();
+		const items = this.getInputData().slice();
 		const returnData: IDataObject[] = [];
 
 		const credentials = this.getCredentials('nextCloudApi');
@@ -583,10 +583,21 @@ export class NextCloud implements INodeType {
 			const responseData = await this.helpers.request(options);
 
 			if (resource === 'file' && operation === 'download') {
-				// TODO: Has to check if it already exists and only add if not
+
-				if (items[i].binary === undefined) {
+				const newItem: INodeExecutionData = {
-					items[i].binary = {};
+					json: items[i].json,
+					binary: {},
+				};
+
+				if (items[i].binary !== undefined) {
+					// Create a shallow copy of the binary data so that the old
+					// data references which do not get changed still stay behind
+					// but the incoming data does not get changed.
+					Object.assign(newItem.binary, items[i].binary);
 				}
+
+				items[i] = newItem;
+
 				const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string;
 
 				items[i].binary![binaryPropertyName] = await this.helpers.prepareBinaryData(responseData, endpoint);

packages/nodes-base/nodes/ReadBinaryFile.node.ts

@@ -55,10 +55,6 @@ export class ReadBinaryFile implements INodeType {
 		const dataPropertyName = this.getNodeParameter('dataPropertyName') as string;
 		const filePath = this.getNodeParameter('filePath') as string;
 
-		if (item.binary === undefined) {
-			item.binary = {};
-		}
-
 		let data;
 		try {
 			data = await fsReadFileAsync(filePath) as Buffer;
@@ -69,9 +65,22 @@ export class ReadBinaryFile implements INodeType {
 
 			throw error;
 		}
-		item.binary[dataPropertyName] = await this.helpers.prepareBinaryData(data, filePath);
 
-		return item;
+		const newItem: INodeExecutionData = {
+			json: item.json,
+			binary: {},
+		};
+
+		if (item.binary !== undefined) {
+			// Create a shallow copy of the binary data so that the old
+			// data references which do not get changed still stay behind
+			// but the incoming data does not get changed.
+			Object.assign(newItem.binary, item.binary);
+		}
+
+		newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(data, filePath);
+
+		return newItem;
 	}
 
 }

packages/nodes-base/nodes/ReadFileFromUrl.node.ts

@@ -69,12 +69,20 @@ export class ReadFileFromUrl implements INodeType {
 		// Get the current item and add the binary data
 		const item = this.getInputData();
 
-		if (!item.binary) {
+		const newItem: INodeExecutionData = {
-			item.binary = {};
+			json: item.json,
+			binary: {},
+		};
+
+		if (item.binary !== undefined) {
+			// Create a shallow copy of the binary data so that the old
+			// data references which do not get changed still stay behind
+			// but the incoming data does not get changed.
+			Object.assign(newItem.binary, item.binary);
 		}
 
-		item.binary[dataPropertyName as string] = await this.helpers.prepareBinaryData(data, fileName);
+		newItem.binary![dataPropertyName as string] = await this.helpers.prepareBinaryData(data, fileName);
 
-		return item;
+		return newItem;
 	}
 }

packages/nodes-base/nodes/Redis/Redis.node.ts

@@ -395,13 +395,9 @@ export class Redis implements INodeType {
 				} else if (['delete', 'get', 'keys', 'set'].includes(operation)) {
 					const items = this.getInputData();
 
-					if (items.length === 0) {
-						items.push({ json: {} });
-					}
-
 					let item: INodeExecutionData;
 					for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
-						item = items[itemIndex];
+						item = { json: {} };
 
 						if (operation === 'delete') {
 							const keyDelete = this.getNodeParameter('key', itemIndex) as string;
@@ -409,8 +405,6 @@ export class Redis implements INodeType {
 							const clientDel = util.promisify(client.del).bind(client);
 							// @ts-ignore
 							await clientDel(keyDelete);
-
-							resolve(this.prepareOutputData(items));
 						} else if (operation === 'get') {
 							const propertyName = this.getNodeParameter('propertyName', itemIndex) as string;
 							const keyGet = this.getNodeParameter('key', itemIndex) as string;
@@ -418,8 +412,6 @@ export class Redis implements INodeType {
 
 							const value = await getValue(client, keyGet, keyType);
 							set(item.json, propertyName, value);
-
-							resolve(this.prepareOutputData(items));
 						} else if (operation === 'keys') {
 							const keyPattern = this.getNodeParameter('keyPattern', itemIndex) as string;
 
@@ -439,18 +431,16 @@ export class Redis implements INodeType {
 							for (const keyName of keys) {
 								set(item.json, keyName, await promises[keyName]);
 							}
-
-							resolve(this.prepareOutputData(items));
 						} else if (operation === 'set') {
 							const keySet = this.getNodeParameter('key', itemIndex) as string;
 							const value = this.getNodeParameter('value', itemIndex) as string;
 							const keyType = this.getNodeParameter('keyType', itemIndex) as string;
 
 							await setValue(client, keySet, value, keyType);
-
-							resolve(this.prepareOutputData(items));
 						}
 					}
+
+					resolve(this.prepareOutputData(items));
 				}
 			});
 		});

packages/nodes-base/nodes/RenameKeys.node.ts

@@ -74,13 +74,27 @@ export class RenameKeys implements INodeType {
 
 		const items = this.getInputData();
 
+		const returnData: INodeExecutionData[] = [];
+
 		let item: INodeExecutionData;
+		let newItem: INodeExecutionData;
 		let renameKeys: IRenameKey[];
 		let value: any; // tslint:disable-line:no-any
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
 			renameKeys = this.getNodeParameter('keys.key', itemIndex, []) as IRenameKey[];
 			item = items[itemIndex];
 
+			// Copy the whole JSON data as data on any level can be renamed
+			newItem = {
+				json: JSON.parse(JSON.stringify(item.json)),
+			};
+
+			if (item.binary !== undefined) {
+				// Reference binary data if any exists. We can reference it
+				// as this nodes does not change it
+				newItem.binary = item.binary;
+			}
+
 			renameKeys.forEach((renameKey) => {
 				if (renameKey.currentKey === '' || renameKey.newKey === '') {
 					// Ignore all which do not have all the values set
@@ -90,12 +104,14 @@ export class RenameKeys implements INodeType {
 				if (value === undefined) {
 					return;
 				}
-				set(item.json, renameKey.newKey, value);
+				set(newItem.json, renameKey.newKey, value);
 
-				unset(item.json, renameKey.currentKey as string);
+				unset(newItem.json, renameKey.currentKey as string);
 			});
+
+			returnData.push(newItem);
 		}
 
-		return this.prepareOutputData(items);
+		return [returnData];
 	}
 }

packages/nodes-base/nodes/Set.node.ts

@@ -115,32 +115,48 @@ export class Set implements INodeType {
 			items.push({json: {}});
 		}
 
+		const returnData: INodeExecutionData[] = [];
+
 		let item: INodeExecutionData;
 		let keepOnlySet: boolean;
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
 			keepOnlySet = this.getNodeParameter('keepOnlySet', itemIndex, []) as boolean;
 			item = items[itemIndex];
 
-			if (keepOnlySet === true) {
+			const newItem: INodeExecutionData = {
-				item.json = {};
+				json: {},
+			};
+
+			if (keepOnlySet !== true) {
+				if (item.binary !== undefined) {
+					// Create a shallow copy of the binary data so that the old
+					// data references which do not get changed still stay behind
+					// but the incoming data does not get changed.
+					newItem.binary = {};
+					Object.assign(newItem.binary, item.binary);
+				}
+
+				newItem.json = JSON.parse(JSON.stringify(item.json));
 			}
 
 			// Add boolean values
 			(this.getNodeParameter('values.boolean', itemIndex, []) as INodeParameters[]).forEach((setItem) => {
-				set(item.json, setItem.name as string, !!setItem.value);
+				set(newItem.json, setItem.name as string, !!setItem.value);
 			});
 
 			// Add number values
 			(this.getNodeParameter('values.number', itemIndex, []) as INodeParameters[]).forEach((setItem) => {
-				set(item.json, setItem.name as string, setItem.value);
+				set(newItem.json, setItem.name as string, setItem.value);
 			});
 
 			// Add string values
 			(this.getNodeParameter('values.string', itemIndex, []) as INodeParameters[]).forEach((setItem) => {
-				set(item.json, setItem.name as string, setItem.value);
+				set(newItem.json, setItem.name as string, setItem.value);
 			});
+
+			returnData.push(newItem);
 		}
 
-		return this.prepareOutputData(items);
+		return this.prepareOutputData(returnData);
 	}
 }

packages/nodes-base/nodes/SplitInBatches.node.ts

@@ -35,7 +35,9 @@ export class SplitInBatches implements INodeType {
 	};
 
 	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][] | null> {
-		const items = this.getInputData();
+		// Get the input data and create a new array so that we can remove
+		// items without a problem
+		const items = this.getInputData().slice();
 
 		const nodeContext = this.getContext('node');
 

packages/nodes-base/nodes/WriteBinaryFile.node.ts

@@ -70,13 +70,22 @@ export class WriteBinaryFile implements INodeType {
 		// Write the file to disk
 		await fsWriteFileAsync(fileName, Buffer.from(item.binary[dataPropertyName].data, BINARY_ENCODING), 'binary');
 
-		if (item.json === undefined) {
+		const newItem: INodeExecutionData = {
-			item.json = {};
+			json: {},
+		};
+		Object.assign(newItem.json, item.json);
+
+		if (item.binary !== undefined) {
+			// Create a shallow copy of the binary data so that the old
+			// data references which do not get changed still stay behind
+			// but the incoming data does not get changed.
+			newItem.binary = {};
+			Object.assign(newItem.binary, item.binary);
 		}
 
 		// Add the file name to data
-		(item.json as IDataObject).fileName = fileName;
+		(newItem.json as IDataObject).fileName = fileName;
 
-		return item;
+		return newItem;
 	}
 }

packages/workflow/src/Workflow.ts

@@ -1045,8 +1045,6 @@ export class Workflow {
 			const returnPromises: Array<Promise<INodeExecutionData>> = [];
 
 			for (let itemIndex = 0; itemIndex < connectionInputData.length; itemIndex++) {
-				// executionData = connectionInputData[itemIndex];
-				// const thisArgs = NodeExecuteFunctions.getExecuteSingleFunctions(this, runData, connectionInputData, inputData, node, itemIndex);
 				const thisArgs = nodeExecuteFunctions.getExecuteSingleFunctions(this, runExecutionData, runIndex, connectionInputData, inputData, node, itemIndex, additionalData, mode);
 
 				returnPromises.push(nodeType.executeSingle!.call(thisArgs));