feat(core): Augment data instead of copying it (#5487)

packages/workflow/src/AugmentObject.ts

@@ -0,0 +1,143 @@
+import type { IDataObject } from './Interfaces';
+import util from 'util';
+
+export function augmentArray<T>(data: T[]): T[] {
+	let newData: unknown[] | undefined = undefined;
+
+	function getData(): unknown[] {
+		if (newData === undefined) {
+			newData = [...data];
+		}
+		return newData;
+	}
+
+	return new Proxy(data, {
+		deleteProperty(target, key: string) {
+			return Reflect.deleteProperty(getData(), key);
+		},
+		get(target, key: string, receiver): unknown {
+			const value = Reflect.get(newData !== undefined ? newData : target, key, receiver) as unknown;
+
+			if (typeof value === 'object') {
+				if (value === null || util.types.isProxy(value)) {
+					return value;
+				}
+
+				newData = getData();
+
+				if (Array.isArray(value)) {
+					Reflect.set(newData, key, augmentArray(value));
+				} else {
+					// eslint-disable-next-line @typescript-eslint/no-use-before-define
+					Reflect.set(newData, key, augmentObject(value as IDataObject));
+				}
+
+				return Reflect.get(newData, key);
+			}
+
+			return value;
+		},
+		getOwnPropertyDescriptor(target, key) {
+			if (newData === undefined) {
+				return Reflect.getOwnPropertyDescriptor(target, key);
+			}
+
+			if (key === 'length') {
+				return Reflect.getOwnPropertyDescriptor(newData, key);
+			}
+			return { configurable: true, enumerable: true };
+		},
+		has(target, key) {
+			return Reflect.has(newData !== undefined ? newData : target, key);
+		},
+		ownKeys(target) {
+			return Reflect.ownKeys(newData !== undefined ? newData : target);
+		},
+		set(target, key: string, newValue: unknown) {
+			if (newValue !== null && typeof newValue === 'object') {
+				// Always proxy all objects. Like that we can check in get simply if it
+				// is a proxy and it does then not matter if it was already there from the
+				// beginning and it got proxied at some point or set later and so theoretically
+				// does not have to get proxied
+				newValue = new Proxy(newValue, {});
+			}
+
+			return Reflect.set(getData(), key, newValue);
+		},
+	});
+}
+
+export function augmentObject<T extends object>(data: T): T {
+	const newData = {} as IDataObject;
+	const deletedProperties: Array<string | symbol> = [];
+
+	return new Proxy(data, {
+		get(target, key: string, receiver): unknown {
+			if (deletedProperties.indexOf(key) !== -1) {
+				return undefined;
+			}
+
+			if (newData[key] !== undefined) {
+				return newData[key];
+			}
+
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+			const value = Reflect.get(target, key, receiver);
+
+			if (value !== null && typeof value === 'object') {
+				if (Array.isArray(value)) {
+					newData[key] = augmentArray(value);
+				} else {
+					newData[key] = augmentObject(value as IDataObject);
+				}
+
+				return newData[key];
+			}
+
+			return value as string;
+		},
+		deleteProperty(target, key: string) {
+			if (key in newData) {
+				delete newData[key];
+			}
+			if (key in target) {
+				deletedProperties.push(key);
+			}
+
+			return true;
+		},
+		set(target, key: string, newValue: unknown) {
+			if (newValue === undefined) {
+				if (key in newData) {
+					delete newData[key];
+				}
+				if (key in target) {
+					deletedProperties.push(key);
+				}
+				return true;
+			}
+
+			newData[key] = newValue as IDataObject;
+
+			const deleteIndex = deletedProperties.indexOf(key);
+			if (deleteIndex !== -1) {
+				deletedProperties.splice(deleteIndex, 1);
+			}
+
+			return true;
+		},
+		ownKeys(target) {
+			return [...new Set([...Reflect.ownKeys(target), ...Object.keys(newData)])].filter(
+				(key) => deletedProperties.indexOf(key) === -1,
+			);
+		},
+
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		getOwnPropertyDescriptor(k) {
+			return {
+				enumerable: true,
+				configurable: true,
+			};
+		},
+	});
+}

packages/workflow/src/WorkflowDataProxy.ts

@@ -28,7 +28,7 @@ import type {
 import * as NodeHelpers from './NodeHelpers';
 import { ExpressionError } from './ExpressionError';
 import type { Workflow } from './Workflow';
-import { deepCopy } from './utils';
+import { augmentArray, augmentObject } from './AugmentObject';
 
 export function isResourceLocatorValue(value: unknown): value is INodeParameterResourceLocator {
 	return Boolean(
@@ -96,11 +96,13 @@ export class WorkflowDataProxy {
 		this.workflow = workflow;
 
 		this.runExecutionData = isScriptingNode(activeNodeName, workflow)
-			? deepCopy(runExecutionData)
+			? runExecutionData !== null
+				? augmentObject(runExecutionData)
+				: null
 			: runExecutionData;
 
 		this.connectionInputData = isScriptingNode(activeNodeName, workflow)
-			? deepCopy(connectionInputData)
+			? augmentArray(connectionInputData)
 			: connectionInputData;
 
 		this.defaultReturnRunIndex = defaultReturnRunIndex;

packages/workflow/test/AugmentObject.test.ts

@@ -0,0 +1,518 @@
+import type { IDataObject } from '@/Interfaces';
+import { augmentArray, augmentObject } from '@/AugmentObject';
+import { deepCopy } from '@/utils';
+
+describe('AugmentObject', () => {
+	describe('augmentArray', () => {
+		test('should work with arrays', () => {
+			const originalObject = [1, 2, 3, 4, null];
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentArray(originalObject);
+
+			expect(augmentedObject.push(5)).toEqual(6);
+			expect(augmentedObject).toEqual([1, 2, 3, 4, null, 5]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.pop()).toEqual(5);
+			expect(augmentedObject).toEqual([1, 2, 3, 4, null]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.shift()).toEqual(1);
+			expect(augmentedObject).toEqual([2, 3, 4, null]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.unshift(1)).toEqual(5);
+			expect(augmentedObject).toEqual([1, 2, 3, 4, null]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.splice(1, 1)).toEqual([2]);
+			expect(augmentedObject).toEqual([1, 3, 4, null]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.slice(1)).toEqual([3, 4, null]);
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.reverse()).toEqual([null, 4, 3, 1]);
+			expect(originalObject).toEqual(copyOriginal);
+		});
+
+		test('should work with arrays on any level', () => {
+			const originalObject = {
+				a: {
+					b: {
+						c: [
+							{
+								a3: {
+									b3: {
+										c3: '03' as string | null,
+									},
+								},
+								aa3: '01',
+							},
+							{
+								a3: {
+									b3: {
+										c3: '13',
+									},
+								},
+								aa3: '11',
+							},
+						],
+					},
+				},
+				aa: [
+					{
+						a3: {
+							b3: '2',
+						},
+						aa3: '1',
+					},
+				],
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			// On first level
+			augmentedObject.aa[0].a3.b3 = '22';
+			expect(augmentedObject.aa[0].a3.b3).toEqual('22');
+			expect(originalObject.aa[0].a3.b3).toEqual('2');
+
+			// Make sure that also array operations as push and length work as expected
+			// On lower levels
+			augmentedObject.a.b.c[0].a3!.b3.c3 = '033';
+			expect(augmentedObject.a.b.c[0].a3!.b3.c3).toEqual('033');
+			expect(originalObject.a.b.c[0].a3!.b3.c3).toEqual('03');
+
+			augmentedObject.a.b.c[1].a3!.b3.c3 = '133';
+			expect(augmentedObject.a.b.c[1].a3!.b3.c3).toEqual('133');
+			expect(originalObject.a.b.c[1].a3!.b3.c3).toEqual('13');
+
+			augmentedObject.a.b.c.push({
+				a3: {
+					b3: {
+						c3: '23',
+					},
+				},
+				aa3: '21',
+			});
+			augmentedObject.a.b.c[2].a3.b3.c3 = '233';
+			expect(augmentedObject.a.b.c[2].a3.b3.c3).toEqual('233');
+
+			augmentedObject.a.b.c[2].a3.b3.c3 = '2333';
+			expect(augmentedObject.a.b.c[2].a3.b3.c3).toEqual('2333');
+
+			augmentedObject.a.b.c[2].a3.b3.c3 = null;
+			expect(augmentedObject.a.b.c[2].a3.b3.c3).toEqual(null);
+
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject.a.b.c.length).toEqual(3);
+
+			expect(augmentedObject.aa).toEqual([
+				{
+					a3: {
+						b3: '22',
+					},
+					aa3: '1',
+				},
+			]);
+
+			expect(augmentedObject.a.b.c).toEqual([
+				{
+					a3: {
+						b3: {
+							c3: '033',
+						},
+					},
+					aa3: '01',
+				},
+				{
+					a3: {
+						b3: {
+							c3: '133',
+						},
+					},
+					aa3: '11',
+				},
+				{
+					a3: {
+						b3: {
+							c3: null,
+						},
+					},
+					aa3: '21',
+				},
+			]);
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						c: [
+							{
+								a3: {
+									b3: {
+										c3: '033',
+									},
+								},
+								aa3: '01',
+							},
+							{
+								a3: {
+									b3: {
+										c3: '133',
+									},
+								},
+								aa3: '11',
+							},
+							{
+								a3: {
+									b3: {
+										c3: null,
+									},
+								},
+								aa3: '21',
+							},
+						],
+					},
+				},
+				aa: [
+					{
+						a3: {
+							b3: '22',
+						},
+						aa3: '1',
+					},
+				],
+			});
+
+			expect(originalObject).toEqual(copyOriginal);
+		});
+	});
+
+	describe('augmentObject', () => {
+		test('should work with simple values on first level', () => {
+			const originalObject: IDataObject = {
+				1: 11,
+				2: '22',
+				a: 111,
+				b: '222',
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			augmentedObject[1] = 911;
+			expect(originalObject[1]).toEqual(11);
+			expect(augmentedObject[1]).toEqual(911);
+
+			augmentedObject[2] = '922';
+			expect(originalObject[2]).toEqual('22');
+			expect(augmentedObject[2]).toEqual('922');
+
+			augmentedObject.a = 9111;
+			expect(originalObject.a).toEqual(111);
+			expect(augmentedObject.a).toEqual(9111);
+
+			augmentedObject.b = '9222';
+			expect(originalObject.b).toEqual('222');
+			expect(augmentedObject.b).toEqual('9222');
+
+			augmentedObject.c = 3;
+
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject).toEqual({
+				1: 911,
+				2: '922',
+				a: 9111,
+				b: '9222',
+				c: 3,
+			});
+		});
+
+		test('should work with simple values on sub-level', () => {
+			const originalObject = {
+				a: {
+					b: {
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			augmentedObject.a.bb = '92';
+			expect(originalObject.a.bb).toEqual('2');
+			expect(augmentedObject.a!.bb!).toEqual('92');
+
+			augmentedObject.a!.b!.cc = '93';
+			expect(originalObject.a.b.cc).toEqual('3');
+			expect(augmentedObject.a!.b!.cc).toEqual('93');
+
+			// @ts-ignore
+			augmentedObject.a!.b!.ccc = {
+				d: '4',
+			};
+
+			// @ts-ignore
+			expect(augmentedObject.a!.b!.ccc).toEqual({ d: '4' });
+
+			// @ts-ignore
+			augmentedObject.a!.b!.ccc.d = '94';
+			// @ts-ignore
+			expect(augmentedObject.a!.b!.ccc.d).toEqual('94');
+
+			expect(originalObject).toEqual(copyOriginal);
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						cc: '93',
+						ccc: {
+							d: '94',
+						},
+					},
+					bb: '92',
+				},
+				aa: '1',
+			});
+		});
+
+		test('should work with complex values on first level', () => {
+			const originalObject = {
+				a: {
+					b: {
+						cc: '3',
+						c2: null,
+					},
+					bb: '2',
+				},
+				aa: '1',
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			augmentedObject.a = { new: 'NEW' };
+			expect(originalObject.a).toEqual({
+				b: {
+					c2: null,
+					cc: '3',
+				},
+				bb: '2',
+			});
+			expect(augmentedObject.a).toEqual({ new: 'NEW' });
+
+			augmentedObject.aa = '11';
+			expect(originalObject.aa).toEqual('1');
+			expect(augmentedObject.aa).toEqual('11');
+
+			augmentedObject.aaa = {
+				bbb: {
+					ccc: '333',
+				},
+			};
+
+			expect(originalObject).toEqual(copyOriginal);
+			expect(augmentedObject).toEqual({
+				a: {
+					new: 'NEW',
+				},
+				aa: '11',
+				aaa: {
+					bbb: {
+						ccc: '333',
+					},
+				},
+			});
+		});
+
+		test('should work with delete and reset', () => {
+			const originalObject = {
+				a: {
+					b: {
+						c: {
+							d: '4' as string | undefined,
+						} as { d?: string; dd?: string } | undefined,
+						cc: '3' as string | undefined,
+					},
+					bb: '2' as string | undefined,
+				},
+				aa: '1' as string | undefined,
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			// Remove multiple values
+			delete augmentedObject.a.b.c!.d;
+			expect(augmentedObject.a.b.c!.d).toEqual(undefined);
+			expect(originalObject.a.b.c!.d).toEqual('4');
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						c: {},
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+
+			delete augmentedObject.a.b.c;
+			expect(augmentedObject.a.b.c).toEqual(undefined);
+			expect(originalObject.a.b.c).toEqual({ d: '4' });
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+
+			// Set deleted values again
+			augmentedObject.a.b.c = { dd: '444' };
+			expect(augmentedObject.a.b.c).toEqual({ dd: '444' });
+			expect(originalObject).toEqual(copyOriginal);
+
+			augmentedObject.a.b.c.d = '44';
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						c: {
+							d: '44',
+							dd: '444',
+						},
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+		});
+
+		// Is almost identical to above test
+		test('should work with setting to undefined and reset', () => {
+			const originalObject = {
+				a: {
+					b: {
+						c: {
+							d: '4' as string | undefined,
+						} as { d?: string; dd?: string } | undefined,
+						cc: '3' as string | undefined,
+					},
+					bb: '2' as string | undefined,
+				},
+				aa: '1' as string | undefined,
+			};
+			const copyOriginal = JSON.parse(JSON.stringify(originalObject));
+
+			const augmentedObject = augmentObject(originalObject);
+
+			// Remove multiple values
+			augmentedObject.a.b.c!.d = undefined;
+			expect(augmentedObject.a.b.c!.d).toEqual(undefined);
+			expect(originalObject.a.b.c!.d).toEqual('4');
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						c: {},
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+
+			augmentedObject.a.b.c = undefined;
+			expect(augmentedObject.a.b.c).toEqual(undefined);
+			expect(originalObject.a.b.c).toEqual({ d: '4' });
+
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+
+			// Set deleted values again
+			augmentedObject.a.b.c = { dd: '444' };
+			expect(augmentedObject.a.b.c).toEqual({ dd: '444' });
+			expect(originalObject).toEqual(copyOriginal);
+
+			augmentedObject.a.b.c.d = '44';
+			expect(augmentedObject).toEqual({
+				a: {
+					b: {
+						c: {
+							d: '44',
+							dd: '444',
+						},
+						cc: '3',
+					},
+					bb: '2',
+				},
+				aa: '1',
+			});
+			expect(originalObject).toEqual(copyOriginal);
+		});
+
+		test('should be faster than doing a deepCopy', () => {
+			const iterations = 100;
+			const originalObject: IDataObject = {
+				a: {
+					b: {
+						c: {
+							d: {
+								e: {
+									f: 12345,
+								},
+							},
+						},
+					},
+				},
+			};
+			for (let i = 0; i < 10; i++) {
+				originalObject[i.toString()] = deepCopy(originalObject);
+			}
+
+			let startTime = new Date().getTime();
+			for (let i = 0; i < iterations; i++) {
+				const augmentedObject = augmentObject(originalObject);
+				for (let i = 0; i < 5000; i++) {
+					augmentedObject.a!.b.c.d.e.f++;
+				}
+			}
+			const timeAugmented = new Date().getTime() - startTime;
+
+			startTime = new Date().getTime();
+			for (let i = 0; i < iterations; i++) {
+				const copiedObject = deepCopy(originalObject);
+				for (let i = 0; i < 5000; i++) {
+					copiedObject.a!.b.c.d.e.f++;
+				}
+			}
+			const timeCopied = new Date().getTime() - startTime;
+
+			expect(timeAugmented).toBeLessThan(timeCopied);
+		});
+	});
+});