feat: Add new expression variables and support for luxon

* 🔨 scaffolding for  and

* 🔨 added autocomplete

* 🔨 N8N-2961-New-expression-variables

* 🔨 added luxon DateTime to expressions and Functions node, replased  with , clean up

* 🔨 added  and , fixed  return values

* 🔨 added tests for new variables

* 🔨 removed unnecessary import

* 🔨 return type fix

* 🔨 working on review, wip

* 🔨 working on review, improved errors, wip

* 🔨 fixed disappearing error message box

* 🔨 excluded variables from function node, added jmespath setup

* :hamer: added $jmsepath to function nodes

* 🔨 replacing proxy with data when using jmespath

* 🔨 renamed function

* 🔨 updated tips to function nodes

* 🔨 fixes for errors messages

* 🔨 review fixes

* 🔨 removed $input and $() from autocomplete

* ⚡ removed comments

* ⚡ Remove unused code

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

package-lock.json

@@ -12581,6 +12581,11 @@
 				"pretty-format": "^26.0.0"
 			}
 		},
+		"@types/jmespath": {
+			"version": "0.15.0",
+			"resolved": "https://registry.npmjs.org/@types/jmespath/-/jmespath-0.15.0.tgz",
+			"integrity": "sha1-SgFikJaoYjHIkaDi3szBX1PJKR0="
+		},
 		"@types/json-diff": {
 			"version": "0.5.2",
 			"resolved": "https://registry.npmjs.org/@types/json-diff/-/json-diff-0.5.2.tgz",
@@ -12656,6 +12661,11 @@
 			"resolved": "https://registry.npmjs.org/@types/lossless-json/-/lossless-json-1.0.1.tgz",
 			"integrity": "sha512-zPE8kmpeL5/6L5gtTQHSOkAW/OSYYNTDRt6/2oEgLO1Zd3Rj5WVDoMloTtLJxQJhZGLGbL4pktKSh3NbzdaWdw=="
 		},
+		"@types/luxon": {
+			"version": "2.0.9",
+			"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.9.tgz",
+			"integrity": "sha512-ZuzIc7aN+i2ZDMWIiSmMdubR9EMMSTdEzF6R+FckP4p6xdnOYKqknTo/k+xXQvciSXlNGIwA4OPU5X7JIFzYdA=="
+		},
 		"@types/mailparser": {
 			"version": "2.7.4",
 			"resolved": "https://registry.npmjs.org/@types/mailparser/-/mailparser-2.7.4.tgz",
@@ -32252,6 +32262,11 @@
 				}
 			}
 		},
+		"luxon": {
+			"version": "2.3.0",
+			"resolved": "https://registry.npmjs.org/luxon/-/luxon-2.3.0.tgz",
+			"integrity": "sha512-gv6jZCV+gGIrVKhO90yrsn8qXPKD8HYZJtrUDSfEbow8Tkw84T9OnCyJhWvnJIaIF/tBuiAjZuQHUt1LddX2mg=="
+		},
 		"macos-release": {
 			"version": "2.5.0",
 			"resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",

packages/editor-ui/package.json

@@ -26,6 +26,7 @@
   },
   "dependencies": {
     "@fontsource/open-sans": "^4.5.0",
+    "luxon": "^2.3.0",
     "n8n-design-system": "~0.13.0",
     "monaco-editor": "^0.29.1",
     "timeago.js": "^4.0.2",
@@ -45,6 +46,7 @@
     "@types/lodash.camelcase": "^4.3.6",
     "@types/lodash.get": "^4.4.6",
     "@types/lodash.set": "^4.3.6",
+    "@types/luxon": "^2.0.9",
     "@types/node": "14.17.27",
     "@types/quill": "^2.0.1",
     "@types/uuid": "^8.3.2",

packages/editor-ui/src/components/CodeEdit.vue

@@ -15,6 +15,7 @@
 
 <script lang="ts">
 import * as monaco from 'monaco-editor/esm/vs/editor/editor.api';
+import {DateTime} from 'luxon';
 
 import { genericHelpers } from '@/components/mixins/genericHelpers';
 import { workflowHelpers } from '@/components/mixins/workflowHelpers';
@@ -176,13 +177,39 @@ export default mixins(
 				const autoCompleteItems = [
 					`function $evaluateExpression(expression: string, itemIndex?: number): any {};`,
 					`function getNodeParameter(parameterName: string, itemIndex: number, fallbackValue?: any): any {};`,
-					`function getWorkflowStaticData(type: string): object {};`,
+					`function getWorkflowStaticData(type: string): {};`,
-					`function $item(itemIndex: number, runIndex?: number) {};`,
+					`function $item(itemIndex: number, runIndex?: number): {};`,
-					`function $items(nodeName?: string, outputIndex?: number, runIndex?: number) {};`,
+					`function $items(nodeName?: string, outputIndex?: number, runIndex?: number): {};`,
 				];
 
-				const baseKeys = ['$env', '$executionId', '$mode', '$parameter', '$position', '$resumeWebhookUrl', '$workflow'];
+				const baseKeys = [
-				const additionalKeys = ['$json', '$binary'];
+					'$env',
+					'$executionId',
+					'$mode',
+					'$parameter',
+					'$resumeWebhookUrl',
+					'$workflow',
+					'$now',
+					'$today',
+					'$thisRunIndex',
+					'DateTime',
+					'Duration',
+					'Interval',
+				];
+
+				const functionItemKeys = [
+					'$json',
+					'$binary',
+					'$position',
+					'$thisItem',
+					'$thisItemIndex',
+				];
+
+				const additionalKeys: string[] = [];
+				if (this.codeAutocomplete === 'functionItem') {
+					additionalKeys.push(...functionItemKeys);
+				}
+
 				if (executedWorkflow && connectionInputData && connectionInputData.length) {
 					baseKeys.push(...additionalKeys);
 				} else {
@@ -216,6 +243,7 @@ export default mixins(
 					} catch(error) {}
 				}
 				autoCompleteItems.push(`const $node = ${JSON.stringify(nodes)}`);
+				autoCompleteItems.push(`function $jmespath(jsonDoc: object, query: string): {};`);
 
 				if (this.codeAutocomplete === 'function') {
 					if (connectionInputData) {

packages/editor-ui/src/components/mixins/pushConnection.ts

@@ -258,6 +258,7 @@ export const pushConnection = mixins(
 							title: 'Problem executing workflow',
 							message: runDataExecutedErrorMessage,
 							type: 'error',
+							duration: 0,
 						});
 					} else {
 						// Workflow did execute without a problem

packages/nodes-base/nodes/Function/Function.node.ts

@@ -36,6 +36,7 @@ export class Function implements INodeType {
 				type: 'string',
 				default: `// Code here will run only once, no matter how many input items there are.
 // More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.function
+// Tip: You can use luxon for dates and $jmespath for querying JSON structures
 
 // Loop over inputs and add a new field called 'myNewField' to the JSON of each one
 for (item of items) {
@@ -151,7 +152,8 @@ return items;`,
 				// Try to find the line number which contains the error and attach to error message
 				const stackLines = error.stack.split('\n');
 				if (stackLines.length > 0) {
-					const lineParts = stackLines[1].split(':');
+					stackLines.shift();
+					const lineParts = stackLines.find((line: string) => line.includes('Function')).split(':');
 					if (lineParts.length > 2) {
 						const lineNumber = lineParts.splice(-2, 1);
 						if (!isNaN(lineNumber)) {

packages/nodes-base/nodes/FunctionItem/FunctionItem.node.ts

@@ -37,6 +37,7 @@ export class FunctionItem implements INodeType {
 				type: 'string',
 				default: `// Code here will run once per input item.
 // More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.functionItem
+// Tip: You can use luxon for dates and $jmespath for querying JSON structures
 
 // Add a new field called 'myNewField' to the JSON of the item
 item.myNewField = 1;
@@ -138,7 +139,8 @@ return item;`,
 						// Try to find the line number which contains the error and attach to error message
 						const stackLines = error.stack.split('\n');
 						if (stackLines.length > 0) {
-							const lineParts = stackLines[1].split(':');
+							stackLines.shift();
+							const lineParts = stackLines.find((line: string) => line.includes('FunctionItem')).split(':');
 							if (lineParts.length > 2) {
 								const lineNumber = lineParts.splice(-2, 1);
 								if (!isNaN(lineNumber)) {

packages/workflow/package.json

@@ -29,8 +29,10 @@
   "devDependencies": {
     "@types/express": "^4.17.6",
     "@types/jest": "^26.0.13",
+    "@types/jmespath": "^0.15.0",
     "@types/lodash.get": "^4.4.6",
     "@types/lodash.merge": "^4.6.6",
+    "@types/luxon": "^2.0.9",
     "@types/lodash.set": "^4.3.6",
     "@types/node": "14.17.27",
     "@types/xml2js": "^0.4.3",
@@ -48,10 +50,12 @@
     "typescript": "~4.3.5"
   },
   "dependencies": {
+    "jmespath": "^0.16.0",
     "lodash.get": "^4.4.2",
     "lodash.isequal": "^4.5.0",
     "lodash.merge": "^4.6.2",
     "lodash.set": "^4.3.2",
+    "luxon": "^2.3.0",
     "riot-tmpl": "^3.0.8",
     "xml2js": "^0.4.23"
   },

packages/workflow/src/Expression.ts

@@ -1,5 +1,7 @@
 // @ts-ignore
 import * as tmpl from 'riot-tmpl';
+import { DateTime, Duration, Interval } from 'luxon';
+
 // eslint-disable-next-line import/no-cycle
 import {
 	INode,
@@ -114,6 +116,12 @@ export class Expression {
 
 		// @ts-ignore
 		data.document = {};
+
+		// @ts-ignore
+		data.DateTime = DateTime;
+		data.Interval = Interval;
+		data.Duration = Duration;
+
 		// @ts-ignore
 		data.constructor = {};
 

packages/workflow/src/Interfaces.ts

@@ -1169,6 +1169,13 @@ export interface IWorkflowDataProxyData {
 	$parameter: any;
 	$position: any;
 	$workflow: any;
+	$: any;
+	$input: any;
+	$thisItem: any;
+	$thisRunIndex: number;
+	$thisItemIndex: number;
+	$now: any;
+	$today: any;
 }
 
 export type IWorkflowDataProxyAdditionalKeys = IDataObject;

packages/workflow/src/WorkflowDataProxy.ts

@@ -6,6 +6,10 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable no-prototype-builtins */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
+
+import { DateTime, Duration, Interval } from 'luxon';
+import * as jmespath from 'jmespath';
+
 // eslint-disable-next-line import/no-cycle
 import {
 	IDataObject,
@@ -224,15 +228,21 @@ export class WorkflowDataProxy {
 			}
 
 			if (!that.runExecutionData.resultData.runData.hasOwnProperty(nodeName)) {
-				throw new Error(`No execution data found for node "${nodeName}"`);
+				if (that.workflow.getNode(nodeName)) {
+					throw new Error(
+						`The node "${nodeName}" hasn't been executed yet, so you can't reference its output data`,
+					);
+				} else {
+					throw new Error(`No node called "${nodeName}" in this workflow`);
+				}
 			}
 
 			runIndex = runIndex === undefined ? that.defaultReturnRunIndex : runIndex;
 			runIndex =
 				runIndex === -1 ? that.runExecutionData.resultData.runData[nodeName].length - 1 : runIndex;
 
-			if (that.runExecutionData.resultData.runData[nodeName].length < runIndex) {
+			if (that.runExecutionData.resultData.runData[nodeName].length <= runIndex) {
-				throw new Error(`No execution data found for run "${runIndex}" of node "${nodeName}"`);
+				throw new Error(`Run ${runIndex} of node "${nodeName}" not found`);
 			}
 
 			const taskData = that.runExecutionData.resultData.runData[nodeName][runIndex].data!;
@@ -264,10 +274,8 @@ export class WorkflowDataProxy {
 				outputIndex = 0;
 			}
 
-			if (taskData.main.length < outputIndex) {
+			if (taskData.main.length <= outputIndex) {
-				throw new Error(
+				throw new Error(`Node "${nodeName}" has no branch with index ${outputIndex}.`);
-					`No data found from "main" input with index "${outputIndex}" via which node is connected with.`,
-				);
 			}
 
 			executionData = taskData.main[outputIndex] as INodeExecutionData[];
@@ -446,7 +454,172 @@ export class WorkflowDataProxy {
 	getDataProxy(): IWorkflowDataProxyData {
 		const that = this;
 
+		const getNodeOutput = (nodeName?: string, branchIndex?: number, runIndex?: number) => {
+			let executionData: INodeExecutionData[];
+
+			if (nodeName === undefined) {
+				executionData = that.connectionInputData;
+			} else {
+				branchIndex = branchIndex || 0;
+				runIndex = runIndex === undefined ? -1 : runIndex;
+				executionData = that.getNodeExecutionData(nodeName, false, branchIndex, runIndex);
+			}
+
+			return executionData;
+		};
+
+		// replacing proxies with the actual data.
+		const jmespathWrapper = (data: IDataObject | IDataObject[], query: string) => {
+			if (!Array.isArray(data) && typeof data === 'object') {
+				return jmespath.search({ ...data }, query);
+			}
+			return jmespath.search(data, query);
+		};
+
 		const base = {
+			$: (nodeName: string) => {
+				if (!nodeName) {
+					throw new Error(`When calling $(), please specify a node`);
+				}
+
+				return new Proxy(
+					{},
+					{
+						get(target, property, receiver) {
+							if (property === 'pairedItem') {
+								return () => {
+									const executionData = getNodeOutput(nodeName, 0, that.runIndex);
+									if (executionData[that.itemIndex]) {
+										return executionData[that.itemIndex];
+									}
+									return undefined;
+								};
+							}
+							if (property === 'item') {
+								return (itemIndex?: number, branchIndex?: number, runIndex?: number) => {
+									if (itemIndex === undefined) {
+										itemIndex = that.itemIndex;
+										branchIndex = 0;
+										runIndex = that.runIndex;
+									}
+									const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
+									if (executionData[itemIndex]) {
+										return executionData[itemIndex];
+									}
+									let errorMessage = '';
+
+									if (branchIndex === undefined && runIndex === undefined) {
+										errorMessage = `
+											No item found at index ${itemIndex}
+											(for node "${nodeName}")`;
+										throw new Error(errorMessage);
+									}
+									if (branchIndex === undefined) {
+										errorMessage = `
+											No item found at index ${itemIndex}
+											in run ${runIndex || that.runIndex}
+											(for node "${nodeName}")`;
+										throw new Error(errorMessage);
+									}
+									if (runIndex === undefined) {
+										errorMessage = `
+											No item found at index ${itemIndex}
+											of branch ${branchIndex || 0}
+											(for node "${nodeName}")`;
+										throw new Error(errorMessage);
+									}
+
+									errorMessage = `
+										No item found at index ${itemIndex}
+										of branch ${branchIndex || 0}
+										in run ${runIndex || that.runIndex}
+										(for node "${nodeName}")`;
+									throw new Error(errorMessage);
+								};
+							}
+							if (property === 'first') {
+								return (branchIndex?: number, runIndex?: number) => {
+									const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
+									if (executionData[0]) return executionData[0];
+									return undefined;
+								};
+							}
+							if (property === 'last') {
+								return (branchIndex?: number, runIndex?: number) => {
+									const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
+									if (!executionData.length) return undefined;
+									if (executionData[executionData.length - 1]) {
+										return executionData[executionData.length - 1];
+									}
+									return undefined;
+								};
+							}
+							if (property === 'all') {
+								return (branchIndex?: number, runIndex?: number) =>
+									getNodeOutput(nodeName, branchIndex, runIndex);
+							}
+							if (property === 'context') {
+								return that.nodeContextGetter(nodeName);
+							}
+							if (property === 'params') {
+								return that.workflow.getNode(nodeName)?.parameters;
+							}
+							return Reflect.get(target, property, receiver);
+						},
+					},
+				);
+			},
+
+			$input: new Proxy(
+				{},
+				{
+					get(target, property, receiver) {
+						if (property === 'thisItem') {
+							return that.connectionInputData[that.itemIndex];
+						}
+						if (property === 'item') {
+							return (itemIndex?: number) => {
+								if (itemIndex === undefined) itemIndex = that.itemIndex;
+								const result = that.connectionInputData;
+								if (result[itemIndex]) {
+									return result[itemIndex];
+								}
+								return undefined;
+							};
+						}
+						if (property === 'first') {
+							return () => {
+								const result = that.connectionInputData;
+								if (result[0]) {
+									return result[0];
+								}
+								return undefined;
+							};
+						}
+						if (property === 'last') {
+							return () => {
+								const result = that.connectionInputData;
+								if (result.length && result[result.length - 1]) {
+									return result[result.length - 1];
+								}
+								return undefined;
+							};
+						}
+						if (property === 'all') {
+							return () => {
+								const result = that.connectionInputData;
+								if (result.length) {
+									return result;
+								}
+								return [];
+							};
+						}
+						return Reflect.get(target, property, receiver);
+					},
+				},
+			),
+
+			$thisItem: that.connectionInputData[that.itemIndex],
 			$binary: {}, // Placeholder
 			$data: {}, // Placeholder
 			$env: this.envGetter(),
@@ -500,6 +673,17 @@ export class WorkflowDataProxy {
 			$runIndex: this.runIndex,
 			$mode: this.mode,
 			$workflow: this.workflowGetter(),
+			$thisRunIndex: this.runIndex,
+			$thisItemIndex: this.itemIndex,
+			$now: DateTime.now(),
+			$today: DateTime.now().set({ hour: 0, minute: 0, second: 0, millisecond: 0 }),
+			$jmespath: jmespathWrapper,
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			DateTime,
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			Interval,
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			Duration,
 			...that.additionalKeys,
 		};
 

packages/workflow/test/WorkflowDataProxy.test.ts

@@ -0,0 +1,201 @@
+import { Workflow, WorkflowDataProxy } from '../src';
+import * as Helpers from './Helpers';
+import {
+	IConnections,
+	INode,
+	INodeExecutionData,
+	IRunExecutionData,
+} from '../src/Interfaces';
+
+describe('WorkflowDataProxy', () => {
+	describe('test data proxy', () => {
+		const nodes: INode[] = [
+			{
+				parameters: {},
+				name: 'Start',
+				type: 'test.set',
+				typeVersion: 1,
+				position: [100, 200],
+			},
+			{
+				parameters: {
+					functionCode:
+						'// Code here will run only once, no matter how many input items there are.\n// More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.function\nconst { DateTime, Duration, Interval } = require("luxon");\n\nconst data = [\n  {\n  "length": 105\n  },\n  {\n  "length": 160\n  },\n  {\n  "length": 121\n  },\n  {\n  "length": 275\n  },\n  {\n  "length": 950\n  },\n];\n\nreturn data.map(fact => ({json: fact}));',
+				},
+				name: 'Function',
+				type: 'test.set',
+				typeVersion: 1,
+				position: [280, 200],
+			},
+			{
+				parameters: {
+					keys: {
+						key: [
+							{
+								currentKey: 'length',
+								newKey: 'data',
+							},
+						],
+					},
+				},
+				name: 'Rename',
+				type: 'test.set',
+				typeVersion: 1,
+				position: [460, 200],
+			},
+		];
+
+		const connections: IConnections = {
+			Start: {
+				main: [
+					[
+						{
+							node: 'Function',
+							type: 'main',
+							index: 0,
+						},
+					],
+				],
+			},
+			Function: {
+				main: [
+					[
+						{
+							node: 'Rename',
+							type: 'main',
+							index: 0,
+						},
+					],
+				],
+			},
+		};
+
+		const runExecutionData: IRunExecutionData = {
+			resultData: {
+				runData: {
+					Function: [
+						{
+							startTime: 1,
+							executionTime: 1,
+							// @ts-ignore
+							data: {
+								main: [
+									[
+										{
+											json: { length: 105 },
+										},
+										{
+											json: { length: 160 },
+										},
+										{
+											json: { length: 121 },
+										},
+										{
+											json: { length: 275 },
+										},
+										{
+											json: { length: 950 },
+										},
+									],
+								],
+							},
+						},
+					],
+					Rename: [
+						{
+							startTime: 1,
+							executionTime: 1,
+							// @ts-ignore
+							data: {
+								main: [
+									[
+										{
+											json: { data: 105 },
+										},
+										{
+											json: { data: 160 },
+										},
+										{
+											json: { data: 121 },
+										},
+										{
+											json: { data: 275 },
+										},
+										{
+											json: { data: 950 },
+										},
+									],
+								],
+							},
+						},
+					],
+				},
+			},
+		};
+
+		const renameNodeConnectionInputData: INodeExecutionData[]  =  [
+			   { json: { length: 105 } },
+			   { json: { length: 160 } },
+			   { json: { length: 121 } },
+			   { json: { length: 275 } },
+			   { json: { length: 950 } }
+			 ]
+
+		const nodeTypes = Helpers.NodeTypes();
+		const workflow = new Workflow({ nodes, connections, active: false, nodeTypes });
+
+		const dataProxy = new WorkflowDataProxy(
+			workflow,
+			runExecutionData,
+			0,
+			0,
+			'Rename',
+			renameNodeConnectionInputData || [],
+			{},
+			'manual',
+			{},
+		);
+		const proxy = dataProxy.getDataProxy();
+
+		test('test $("NodeName").all()', () => {
+			expect(proxy.$('Rename').all()[1].json.data).toEqual(160);
+		});
+		test('test $("NodeName").all() length', () => {
+			expect(proxy.$('Rename').all().length).toEqual(5);
+		});
+		test('test $("NodeName").item()', () => {
+			expect(proxy.$('Rename').item().json.data).toEqual(105);
+		});
+		test('test $("NodeName").item(2)', () => {
+			expect(proxy.$('Rename').item(2).json.data).toEqual(121);
+		});
+		test('test $("NodeName").first()', () => {
+			expect(proxy.$('Rename').first().json.data).toEqual(105);
+		});
+		test('test $("NodeName").last()', () => {
+			expect(proxy.$('Rename').last().json.data).toEqual(950);
+		});
+
+		test('test $input.all()', () => {
+			expect(proxy.$input.all()[1].json.length).toEqual(160);
+		});
+		test('test $input.all() length', () => {
+			expect(proxy.$input.all().length).toEqual(5);
+		});
+		test('test $input.item()', () => {
+			expect(proxy.$input.item().json.length).toEqual(105);
+		});
+		test('test $thisItem', () => {
+			expect(proxy.$thisItem.json.length).toEqual(105);
+		});
+		test('test $input.item(2)', () => {
+			expect(proxy.$input.item(2).json.length).toEqual(121);
+		});
+		test('test $input.first()', () => {
+			expect(proxy.$input.first().json.length).toEqual(105);
+		});
+		test('test $input.last()', () => {
+			expect(proxy.$input.last().json.length).toEqual(950);
+		});
+	});
+});