mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-12-23 11:44:06 -08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat: Add new expression variables and support for luxon

Browse source 
* 🔨 scaffolding for  and

* 🔨 added autocomplete

* 🔨 N8N-2961-New-expression-variables

* 🔨 added luxon DateTime to expressions and Functions node, replased  with , clean up

* 🔨 added  and , fixed  return values

* 🔨 added tests for new variables

* 🔨 removed unnecessary import

* 🔨 return type fix

* 🔨 working on review, wip

* 🔨 working on review, improved errors, wip

* 🔨 fixed disappearing error message box

* 🔨 excluded variables from function node, added jmespath setup

* :hamer: added $jmsepath to function nodes

* 🔨 replacing proxy with data when using jmespath

* 🔨 renamed function

* 🔨 updated tips to function nodes

* 🔨 fixes for errors messages

* 🔨 review fixes

* 🔨 removed $input and $() from autocomplete

* ⚡ removed comments

* ⚡ Remove unused code

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Michael Kret 2022-03-13 11:34:44 +02:00 committed by GitHub
parent a957142a70
commit e8500e6937
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 468 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
package-lock.json generated
View file

@ -12581,6 +12581,11 @@
"pretty-format": "^26.0.0"
}
},
"@types/jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/@types/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-SgFikJaoYjHIkaDi3szBX1PJKR0="
},
"@types/json-diff": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/@types/json-diff/-/json-diff-0.5.2.tgz",
@ -12656,6 +12661,11 @@
"resolved": "https://registry.npmjs.org/@types/lossless-json/-/lossless-json-1.0.1.tgz",
"integrity": "sha512-zPE8kmpeL5/6L5gtTQHSOkAW/OSYYNTDRt6/2oEgLO1Zd3Rj5WVDoMloTtLJxQJhZGLGbL4pktKSh3NbzdaWdw=="
},
"@types/luxon": {
"version": "2.0.9",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.9.tgz",
"integrity": "sha512-ZuzIc7aN+i2ZDMWIiSmMdubR9EMMSTdEzF6R+FckP4p6xdnOYKqknTo/k+xXQvciSXlNGIwA4OPU5X7JIFzYdA=="
},
"@types/mailparser": {
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/@types/mailparser/-/mailparser-2.7.4.tgz",
@ -32252,6 +32262,11 @@
}
}
},
"luxon": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-2.3.0.tgz",
"integrity": "sha512-gv6jZCV+gGIrVKhO90yrsn8qXPKD8HYZJtrUDSfEbow8Tkw84T9OnCyJhWvnJIaIF/tBuiAjZuQHUt1LddX2mg=="
},
"macos-release": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",

2
packages/editor-ui/package.json
View file

@ -26,6 +26,7 @@
},
"dependencies": {
"@fontsource/open-sans": "^4.5.0",
"luxon": "^2.3.0",
"n8n-design-system": "~0.13.0",
"monaco-editor": "^0.29.1",
"timeago.js": "^4.0.2",
@ -45,6 +46,7 @@
"@types/lodash.camelcase": "^4.3.6",
"@types/lodash.get": "^4.4.6",
"@types/lodash.set": "^4.3.6",
"@types/luxon": "^2.0.9",
"@types/node": "14.17.27",
"@types/quill": "^2.0.1",
"@types/uuid": "^8.3.2",

38
packages/editor-ui/src/components/CodeEdit.vue
View file

@ -15,6 +15,7 @@
<script lang="ts">
import * as monaco from 'monaco-editor/esm/vs/editor/editor.api';
import {DateTime} from 'luxon';
import { genericHelpers } from '@/components/mixins/genericHelpers';
import { workflowHelpers } from '@/components/mixins/workflowHelpers';
@ -176,13 +177,39 @@ export default mixins(
const autoCompleteItems = [
`function $evaluateExpression(expression: string, itemIndex?: number): any {};`,
`function getNodeParameter(parameterName: string, itemIndex: number, fallbackValue?: any): any {};`,
`function getWorkflowStaticData(type: string): object {};`,
`function $item(itemIndex: number, runIndex?: number) {};`,
`function $items(nodeName?: string, outputIndex?: number, runIndex?: number) {};`,
`function getWorkflowStaticData(type: string): {};`,
`function $item(itemIndex: number, runIndex?: number): {};`,
`function $items(nodeName?: string, outputIndex?: number, runIndex?: number): {};`,
];
const baseKeys = ['$env', '$executionId', '$mode', '$parameter', '$position', '$resumeWebhookUrl', '$workflow'];
const additionalKeys = ['$json', '$binary'];
const baseKeys = [
'$env',
'$executionId',
'$mode',
'$parameter',
'$resumeWebhookUrl',
'$workflow',
'$now',
'$today',
'$thisRunIndex',
'DateTime',
'Duration',
'Interval',
];
const functionItemKeys = [
'$json',
'$binary',
'$position',
'$thisItem',
'$thisItemIndex',
];
const additionalKeys: string[] = [];
if (this.codeAutocomplete === 'functionItem') {
additionalKeys.push(...functionItemKeys);
}
if (executedWorkflow && connectionInputData && connectionInputData.length) {
baseKeys.push(...additionalKeys);
} else {
@ -216,6 +243,7 @@ export default mixins(
} catch(error) {}
}
autoCompleteItems.push(`const $node = ${JSON.stringify(nodes)}`);
autoCompleteItems.push(`function $jmespath(jsonDoc: object, query: string): {};`);
if (this.codeAutocomplete === 'function') {
if (connectionInputData) {

1
packages/editor-ui/src/components/mixins/pushConnection.ts
View file

@ -258,6 +258,7 @@ export const pushConnection = mixins(
title: 'Problem executing workflow',
message: runDataExecutedErrorMessage,
type: 'error',
duration: 0,
});
} else {
// Workflow did execute without a problem

4
packages/nodes-base/nodes/Function/Function.node.ts
View file

@ -36,6 +36,7 @@ export class Function implements INodeType {
type: 'string',
default: `// Code here will run only once, no matter how many input items there are.
// More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.function
// Tip: You can use luxon for dates and $jmespath for querying JSON structures
// Loop over inputs and add a new field called 'myNewField' to the JSON of each one
for (item of items) {
@ -151,7 +152,8 @@ return items;`,
// Try to find the line number which contains the error and attach to error message
const stackLines = error.stack.split('\n');
if (stackLines.length > 0) {
const lineParts = stackLines[1].split(':');
stackLines.shift();
const lineParts = stackLines.find((line: string) => line.includes('Function')).split(':');
if (lineParts.length > 2) {
const lineNumber = lineParts.splice(-2, 1);
if (!isNaN(lineNumber)) {

4
packages/nodes-base/nodes/FunctionItem/FunctionItem.node.ts
View file

@ -37,6 +37,7 @@ export class FunctionItem implements INodeType {
type: 'string',
default: `// Code here will run once per input item.
// More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.functionItem
// Tip: You can use luxon for dates and $jmespath for querying JSON structures
// Add a new field called 'myNewField' to the JSON of the item
item.myNewField = 1;
@ -138,7 +139,8 @@ return item;`,
// Try to find the line number which contains the error and attach to error message
const stackLines = error.stack.split('\n');
if (stackLines.length > 0) {
const lineParts = stackLines[1].split(':');
stackLines.shift();
const lineParts = stackLines.find((line: string) => line.includes('FunctionItem')).split(':');
if (lineParts.length > 2) {
const lineNumber = lineParts.splice(-2, 1);
if (!isNaN(lineNumber)) {

4
packages/workflow/package.json
View file

@ -29,8 +29,10 @@
"devDependencies": {
"@types/express": "^4.17.6",
"@types/jest": "^26.0.13",
"@types/jmespath": "^0.15.0",
"@types/lodash.get": "^4.4.6",
"@types/lodash.merge": "^4.6.6",
"@types/luxon": "^2.0.9",
"@types/lodash.set": "^4.3.6",
"@types/node": "14.17.27",
"@types/xml2js": "^0.4.3",
@ -48,10 +50,12 @@
"typescript": "~4.3.5"
},
"dependencies": {
"jmespath": "^0.16.0",
"lodash.get": "^4.4.2",
"lodash.isequal": "^4.5.0",
"lodash.merge": "^4.6.2",
"lodash.set": "^4.3.2",
"luxon": "^2.3.0",
"riot-tmpl": "^3.0.8",
"xml2js": "^0.4.23"
},

8
packages/workflow/src/Expression.ts
View file

@ -1,5 +1,7 @@
// @ts-ignore
import * as tmpl from 'riot-tmpl';
import { DateTime, Duration, Interval } from 'luxon';
// eslint-disable-next-line import/no-cycle
import {
INode,
@ -114,6 +116,12 @@ export class Expression {
// @ts-ignore
data.document = {};
// @ts-ignore
data.DateTime = DateTime;
data.Interval = Interval;
data.Duration = Duration;
// @ts-ignore
data.constructor = {};

7
packages/workflow/src/Interfaces.ts
View file

@ -1169,6 +1169,13 @@ export interface IWorkflowDataProxyData {
$parameter: any;
$position: any;
$workflow: any;
$: any;
$input: any;
$thisItem: any;
$thisRunIndex: number;
$thisItemIndex: number;
$now: any;
$today: any;
}
export type IWorkflowDataProxyAdditionalKeys = IDataObject;

198
packages/workflow/src/WorkflowDataProxy.ts
View file

@ -6,6 +6,10 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable no-prototype-builtins */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import { DateTime, Duration, Interval } from 'luxon';
import * as jmespath from 'jmespath';
// eslint-disable-next-line import/no-cycle
import {
IDataObject,
@ -224,15 +228,21 @@ export class WorkflowDataProxy {
}
if (!that.runExecutionData.resultData.runData.hasOwnProperty(nodeName)) {
throw new Error(`No execution data found for node "${nodeName}"`);
if (that.workflow.getNode(nodeName)) {
throw new Error(
`The node "${nodeName}" hasn't been executed yet, so you can't reference its output data`,
);
} else {
throw new Error(`No node called "${nodeName}" in this workflow`);
}
}
runIndex = runIndex === undefined ? that.defaultReturnRunIndex : runIndex;
runIndex =
runIndex === -1 ? that.runExecutionData.resultData.runData[nodeName].length - 1 : runIndex;
if (that.runExecutionData.resultData.runData[nodeName].length < runIndex) {
throw new Error(`No execution data found for run "${runIndex}" of node "${nodeName}"`);
if (that.runExecutionData.resultData.runData[nodeName].length <= runIndex) {
throw new Error(`Run ${runIndex} of node "${nodeName}" not found`);
}
const taskData = that.runExecutionData.resultData.runData[nodeName][runIndex].data!;
@ -264,10 +274,8 @@ export class WorkflowDataProxy {
outputIndex = 0;
}
if (taskData.main.length < outputIndex) {
throw new Error(
`No data found from "main" input with index "${outputIndex}" via which node is connected with.`,
);
if (taskData.main.length <= outputIndex) {
throw new Error(`Node "${nodeName}" has no branch with index ${outputIndex}.`);
}
executionData = taskData.main[outputIndex] as INodeExecutionData[];
@ -446,7 +454,172 @@ export class WorkflowDataProxy {
getDataProxy(): IWorkflowDataProxyData {
const that = this;
const getNodeOutput = (nodeName?: string, branchIndex?: number, runIndex?: number) => {
let executionData: INodeExecutionData[];
if (nodeName === undefined) {
executionData = that.connectionInputData;
} else {
branchIndex = branchIndex || 0;
runIndex = runIndex === undefined ? -1 : runIndex;
executionData = that.getNodeExecutionData(nodeName, false, branchIndex, runIndex);
}
return executionData;
};
// replacing proxies with the actual data.
const jmespathWrapper = (data: IDataObject | IDataObject[], query: string) => {
if (!Array.isArray(data) && typeof data === 'object') {
return jmespath.search({ ...data }, query);
}
return jmespath.search(data, query);
};
const base = {
$: (nodeName: string) => {
if (!nodeName) {
throw new Error(`When calling $(), please specify a node`);
}
return new Proxy(
{},
{
get(target, property, receiver) {
if (property === 'pairedItem') {
return () => {
const executionData = getNodeOutput(nodeName, 0, that.runIndex);
if (executionData[that.itemIndex]) {
return executionData[that.itemIndex];
}
return undefined;
};
}
if (property === 'item') {
return (itemIndex?: number, branchIndex?: number, runIndex?: number) => {
if (itemIndex === undefined) {
itemIndex = that.itemIndex;
branchIndex = 0;
runIndex = that.runIndex;
}
const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
if (executionData[itemIndex]) {
return executionData[itemIndex];
}
let errorMessage = '';
if (branchIndex === undefined && runIndex === undefined) {
errorMessage = `
No item found at index ${itemIndex}
(for node "${nodeName}")`;
throw new Error(errorMessage);
}
if (branchIndex === undefined) {
errorMessage = `
No item found at index ${itemIndex}
in run ${runIndex || that.runIndex}
(for node "${nodeName}")`;
throw new Error(errorMessage);
}
if (runIndex === undefined) {
errorMessage = `
No item found at index ${itemIndex}
of branch ${branchIndex || 0}
(for node "${nodeName}")`;
throw new Error(errorMessage);
}
errorMessage = `
No item found at index ${itemIndex}
of branch ${branchIndex || 0}
in run ${runIndex || that.runIndex}
(for node "${nodeName}")`;
throw new Error(errorMessage);
};
}
if (property === 'first') {
return (branchIndex?: number, runIndex?: number) => {
const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
if (executionData[0]) return executionData[0];
return undefined;
};
}
if (property === 'last') {
return (branchIndex?: number, runIndex?: number) => {
const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
if (!executionData.length) return undefined;
if (executionData[executionData.length - 1]) {
return executionData[executionData.length - 1];
}
return undefined;
};
}
if (property === 'all') {
return (branchIndex?: number, runIndex?: number) =>
getNodeOutput(nodeName, branchIndex, runIndex);
}
if (property === 'context') {
return that.nodeContextGetter(nodeName);
}
if (property === 'params') {
return that.workflow.getNode(nodeName)?.parameters;
}
return Reflect.get(target, property, receiver);
},
},
);
},
$input: new Proxy(
{},
{
get(target, property, receiver) {
if (property === 'thisItem') {
return that.connectionInputData[that.itemIndex];
}
if (property === 'item') {
return (itemIndex?: number) => {
if (itemIndex === undefined) itemIndex = that.itemIndex;
const result = that.connectionInputData;
if (result[itemIndex]) {
return result[itemIndex];
}
return undefined;
};
}
if (property === 'first') {
return () => {
const result = that.connectionInputData;
if (result[0]) {
return result[0];
}
return undefined;
};
}
if (property === 'last') {
return () => {
const result = that.connectionInputData;
if (result.length && result[result.length - 1]) {
return result[result.length - 1];
}
return undefined;
};
}
if (property === 'all') {
return () => {
const result = that.connectionInputData;
if (result.length) {
return result;
}
return [];
};
}
return Reflect.get(target, property, receiver);
},
},
),
$thisItem: that.connectionInputData[that.itemIndex],
$binary: {}, // Placeholder
$data: {}, // Placeholder
$env: this.envGetter(),
@ -500,6 +673,17 @@ export class WorkflowDataProxy {
$runIndex: this.runIndex,
$mode: this.mode,
$workflow: this.workflowGetter(),
$thisRunIndex: this.runIndex,
$thisItemIndex: this.itemIndex,
$now: DateTime.now(),
$today: DateTime.now().set({ hour: 0, minute: 0, second: 0, millisecond: 0 }),
$jmespath: jmespathWrapper,
// eslint-disable-next-line @typescript-eslint/naming-convention
DateTime,
// eslint-disable-next-line @typescript-eslint/naming-convention
Interval,
// eslint-disable-next-line @typescript-eslint/naming-convention
Duration,
...that.additionalKeys,
};

201
packages/workflow/test/WorkflowDataProxy.test.ts Normal file
View file

@ -0,0 +1,201 @@
import { Workflow, WorkflowDataProxy } from '../src';
import * as Helpers from './Helpers';
import {
IConnections,
INode,
INodeExecutionData,
IRunExecutionData,
} from '../src/Interfaces';
describe('WorkflowDataProxy', () => {
describe('test data proxy', () => {
const nodes: INode[] = [
{
parameters: {},
name: 'Start',
type: 'test.set',
typeVersion: 1,
position: [100, 200],
},
{
parameters: {
functionCode:
'// Code here will run only once, no matter how many input items there are.\n// More info and help: https://docs.n8n.io/nodes/n8n-nodes-base.function\nconst { DateTime, Duration, Interval } = require("luxon");\n\nconst data = [\n {\n "length": 105\n },\n {\n "length": 160\n },\n {\n "length": 121\n },\n {\n "length": 275\n },\n {\n "length": 950\n },\n];\n\nreturn data.map(fact => ({json: fact}));',
},
name: 'Function',
type: 'test.set',
typeVersion: 1,
position: [280, 200],
},
{
parameters: {
keys: {
key: [
{
currentKey: 'length',
newKey: 'data',
},
],
},
},
name: 'Rename',
type: 'test.set',
typeVersion: 1,
position: [460, 200],
},
];
const connections: IConnections = {
Start: {
main: [
[
{
node: 'Function',
type: 'main',
index: 0,
},
],
],
},
Function: {
main: [
[
{
node: 'Rename',
type: 'main',
index: 0,
},
],
],
},
};
const runExecutionData: IRunExecutionData = {
resultData: {
runData: {
Function: [
{
startTime: 1,
executionTime: 1,
// @ts-ignore
data: {
main: [
[
{
json: { length: 105 },
},
{
json: { length: 160 },
},
{
json: { length: 121 },
},
{
json: { length: 275 },
},
{
json: { length: 950 },
},
],
],
},
},
],
Rename: [
{
startTime: 1,
executionTime: 1,
// @ts-ignore
data: {
main: [
[
{
json: { data: 105 },
},
{
json: { data: 160 },
},
{
json: { data: 121 },
},
{
json: { data: 275 },
},
{
json: { data: 950 },
},
],
],
},
},
],
},
},
};
const renameNodeConnectionInputData: INodeExecutionData[] = [
{ json: { length: 105 } },
{ json: { length: 160 } },
{ json: { length: 121 } },
{ json: { length: 275 } },
{ json: { length: 950 } }
]
const nodeTypes = Helpers.NodeTypes();
const workflow = new Workflow({ nodes, connections, active: false, nodeTypes });
const dataProxy = new WorkflowDataProxy(
workflow,
runExecutionData,
0,
0,
'Rename',
renameNodeConnectionInputData || [],
{},
'manual',
{},
);
const proxy = dataProxy.getDataProxy();
test('test $("NodeName").all()', () => {
expect(proxy.$('Rename').all()[1].json.data).toEqual(160);
});
test('test $("NodeName").all() length', () => {
expect(proxy.$('Rename').all().length).toEqual(5);
});
test('test $("NodeName").item()', () => {
expect(proxy.$('Rename').item().json.data).toEqual(105);
});
test('test $("NodeName").item(2)', () => {
expect(proxy.$('Rename').item(2).json.data).toEqual(121);
});
test('test $("NodeName").first()', () => {
expect(proxy.$('Rename').first().json.data).toEqual(105);
});
test('test $("NodeName").last()', () => {
expect(proxy.$('Rename').last().json.data).toEqual(950);
});
test('test $input.all()', () => {
expect(proxy.$input.all()[1].json.length).toEqual(160);
});
test('test $input.all() length', () => {
expect(proxy.$input.all().length).toEqual(5);
});
test('test $input.item()', () => {
expect(proxy.$input.item().json.length).toEqual(105);
});
test('test $thisItem', () => {
expect(proxy.$thisItem.json.length).toEqual(105);
});
test('test $input.item(2)', () => {
expect(proxy.$input.item(2).json.length).toEqual(121);
});
test('test $input.first()', () => {
expect(proxy.$input.first().json.length).toEqual(105);
});
test('test $input.last()', () => {
expect(proxy.$input.last().json.length).toEqual(950);
});
});
});