feat: Add support for $env in the js task runner (no-changelog) (#11177)
Some checks are pending
Test Master / install-and-build (push) Waiting to run
Test Master / Unit tests (18.x) (push) Blocked by required conditions
Test Master / Unit tests (20.x) (push) Blocked by required conditions
Test Master / Unit tests (22.4) (push) Blocked by required conditions
Test Master / Lint (push) Blocked by required conditions
Test Master / Notify Slack on failure (push) Blocked by required conditions
Benchmark Docker Image CI / build (push) Waiting to run

This commit is contained in:
Tomi Turtiainen 2024-10-09 17:31:45 +03:00 committed by GitHub
parent 939c0dfc4c
commit e94cda3837
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 445 additions and 88 deletions

View file

@ -26,5 +26,8 @@
"n8n-core": "workspace:*",
"nanoid": "^3.3.6",
"ws": "^8.18.0"
},
"devDependencies": {
"luxon": "catalog:"
}
}

View file

@ -1,3 +1,4 @@
import { DateTime } from 'luxon';
import type { CodeExecutionMode, IDataObject } from 'n8n-workflow';
import { ValidationError } from '@/js-task-runner/errors/validation-error';
@ -30,6 +31,36 @@ describe('JsTaskRunner', () => {
jest.restoreAllMocks();
});
const executeForAllItems = async ({
code,
inputItems,
settings,
}: { code: string; inputItems: IDataObject[]; settings?: Partial<JSExecSettings> }) => {
return await execTaskWithParams({
task: newTaskWithSettings({
code,
nodeMode: 'runOnceForAllItems',
...settings,
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson)),
});
};
const executeForEachItem = async ({
code,
inputItems,
settings,
}: { code: string; inputItems: IDataObject[]; settings?: Partial<JSExecSettings> }) => {
return await execTaskWithParams({
task: newTaskWithSettings({
code,
nodeMode: 'runOnceForEachItem',
...settings,
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson)),
});
};
describe('console', () => {
test.each<[CodeExecutionMode]>([['runOnceForAllItems'], ['runOnceForEachItem']])(
'should make an rpc call for console log in %s mode',
@ -52,22 +83,178 @@ describe('JsTaskRunner', () => {
);
});
describe('runOnceForAllItems', () => {
const executeForAllItems = async ({
code,
inputItems,
settings,
}: { code: string; inputItems: IDataObject[]; settings?: Partial<JSExecSettings> }) => {
return await execTaskWithParams({
task: newTaskWithSettings({
code,
nodeMode: 'runOnceForAllItems',
...settings,
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson)),
describe('built-in methods and variables available in the context', () => {
const inputItems = [{ a: 1 }];
const testExpressionForAllItems = async (
expression: string,
expected: IDataObject | string | number | boolean,
) => {
const needsWrapping = typeof expected !== 'object';
const outcome = await executeForAllItems({
code: needsWrapping ? `return { val: ${expression} }` : `return ${expression}`,
inputItems,
});
expect(outcome.result).toEqual([wrapIntoJson(needsWrapping ? { val: expected } : expected)]);
};
const testExpressionForEachItem = async (
expression: string,
expected: IDataObject | string | number | boolean,
) => {
const needsWrapping = typeof expected !== 'object';
const outcome = await executeForEachItem({
code: needsWrapping ? `return { val: ${expression} }` : `return ${expression}`,
inputItems,
});
expect(outcome.result).toEqual([
withPairedItem(0, wrapIntoJson(needsWrapping ? { val: expected } : expected)),
]);
};
const testGroups = {
// https://docs.n8n.io/code/builtin/current-node-input/
'current node input': [
['$input.first()', inputItems[0]],
['$input.last()', inputItems[inputItems.length - 1]],
['$input.params', { manualTriggerParam: 'empty' }],
],
// https://docs.n8n.io/code/builtin/output-other-nodes/
'output of other nodes': [
['$("Trigger").first()', inputItems[0]],
['$("Trigger").last()', inputItems[inputItems.length - 1]],
['$("Trigger").params', { manualTriggerParam: 'empty' }],
],
// https://docs.n8n.io/code/builtin/date-time/
'date and time': [
['$now', expect.any(DateTime)],
['$today', expect.any(DateTime)],
['{dt: DateTime}', { dt: expect.any(Function) }],
],
// https://docs.n8n.io/code/builtin/jmespath/
JMESPath: [['{ val: $jmespath([{ f: 1 },{ f: 2 }], "[*].f") }', { val: [1, 2] }]],
// https://docs.n8n.io/code/builtin/n8n-metadata/
'n8n metadata': [
[
'$execution',
{
id: 'exec-id',
mode: 'test',
resumeFormUrl: 'http://formWaitingBaseUrl/exec-id',
resumeUrl: 'http://webhookWaitingBaseUrl/exec-id',
customData: {
get: expect.any(Function),
getAll: expect.any(Function),
set: expect.any(Function),
setAll: expect.any(Function),
},
},
],
['$("Trigger").isExecuted', true],
['$nodeVersion', 2],
['$prevNode.name', 'Trigger'],
['$prevNode.outputIndex', 0],
['$runIndex', 0],
['{ wf: $workflow }', { wf: { active: true, id: '1', name: 'Test Workflow' } }],
['$vars', { var: 'value' }],
],
};
for (const [groupName, tests] of Object.entries(testGroups)) {
describe(`${groupName} runOnceForAllItems`, () => {
test.each(tests)(
'should have the %s available in the context',
async (expression, expected) => {
await testExpressionForAllItems(expression, expected);
},
);
});
describe(`${groupName} runOnceForEachItem`, () => {
test.each(tests)(
'should have the %s available in the context',
async (expression, expected) => {
await testExpressionForEachItem(expression, expected);
},
);
});
}
describe('$env', () => {
it('should have the env available in context when access has not been blocked', async () => {
const outcome = await execTaskWithParams({
task: newTaskWithSettings({
code: 'return { val: $env.VAR1 }',
nodeMode: 'runOnceForAllItems',
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), {
envProviderState: {
isEnvAccessBlocked: false,
isProcessAvailable: true,
env: { VAR1: 'value' },
},
}),
});
expect(outcome.result).toEqual([wrapIntoJson({ val: 'value' })]);
});
it('should be possible to access env if it has been blocked', async () => {
await expect(
execTaskWithParams({
task: newTaskWithSettings({
code: 'return { val: $env.VAR1 }',
nodeMode: 'runOnceForAllItems',
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), {
envProviderState: {
isEnvAccessBlocked: true,
isProcessAvailable: true,
env: { VAR1: 'value' },
},
}),
}),
).rejects.toThrow('access to env vars denied');
});
it('should not be possible to iterate $env', async () => {
const outcome = await execTaskWithParams({
task: newTaskWithSettings({
code: 'return Object.values($env).concat(Object.keys($env))',
nodeMode: 'runOnceForAllItems',
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), {
envProviderState: {
isEnvAccessBlocked: false,
isProcessAvailable: true,
env: { VAR1: '1', VAR2: '2', VAR3: '3' },
},
}),
});
expect(outcome.result).toEqual([]);
});
it("should not expose task runner's env variables even if no env state is received", async () => {
process.env.N8N_RUNNERS_N8N_URI = 'http://127.0.0.1:5679';
const outcome = await execTaskWithParams({
task: newTaskWithSettings({
code: 'return { val: $env.N8N_RUNNERS_N8N_URI }',
nodeMode: 'runOnceForAllItems',
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson), {
envProviderState: undefined,
}),
});
expect(outcome.result).toEqual([wrapIntoJson({ val: undefined })]);
});
});
});
describe('runOnceForAllItems', () => {
describe('continue on fail', () => {
it('should return an item with the error if continueOnFail is true', async () => {
const outcome = await executeForAllItems({
@ -181,21 +368,6 @@ describe('JsTaskRunner', () => {
});
describe('runForEachItem', () => {
const executeForEachItem = async ({
code,
inputItems,
settings,
}: { code: string; inputItems: IDataObject[]; settings?: Partial<JSExecSettings> }) => {
return await execTaskWithParams({
task: newTaskWithSettings({
code,
nodeMode: 'runOnceForEachItem',
...settings,
}),
taskData: newAllCodeTaskData(inputItems.map(wrapIntoJson)),
});
};
describe('continue on fail', () => {
it('should return an item with the error if continueOnFail is true', async () => {
const outcome = await executeForEachItem({

View file

@ -65,6 +65,9 @@ export const newAllCodeTaskData = (
const manualTriggerNode = newNode({
name: 'Trigger',
type: 'n8n-nodes-base.manualTrigger',
parameters: {
manualTriggerParam: 'empty',
},
});
return {
@ -116,15 +119,32 @@ export const newAllCodeTaskData = (
siblingParameters: {},
mode: 'manual',
selfData: {},
envProviderState: {
env: {},
isEnvAccessBlocked: true,
isProcessAvailable: true,
},
additionalData: {
formWaitingBaseUrl: '',
executionId: 'exec-id',
instanceBaseUrl: '',
restartExecutionId: '',
restApiUrl: '',
webhookBaseUrl: '',
webhookTestBaseUrl: '',
webhookWaitingBaseUrl: '',
variables: {},
formWaitingBaseUrl: 'http://formWaitingBaseUrl',
webhookBaseUrl: 'http://webhookBaseUrl',
webhookTestBaseUrl: 'http://webhookTestBaseUrl',
webhookWaitingBaseUrl: 'http://webhookWaitingBaseUrl',
variables: {
var: 'value',
},
},
executeData: {
node: codeNode,
data: {
main: [codeNodeInputData],
},
source: {
main: [{ previousNode: manualTriggerNode.name }],
},
},
...opts,
};

View file

@ -17,6 +17,7 @@ import type {
INodeParameters,
IRunExecutionData,
WorkflowExecuteMode,
EnvProviderState,
} from 'n8n-workflow';
import * as a from 'node:assert';
import { runInNewContext, type Context } from 'node:vm';
@ -63,6 +64,7 @@ export interface AllCodeTaskData {
connectionInputData: INodeExecutionData[];
siblingParameters: INodeParameters;
mode: WorkflowExecuteMode;
envProviderState?: EnvProviderState;
executeData?: IExecuteData;
defaultReturnRunIndex: number;
selfData: IDataObject;
@ -262,6 +264,13 @@ export class JsTaskRunner extends TaskRunner {
allData.defaultReturnRunIndex,
allData.selfData,
allData.contextNodeName,
// Make sure that even if we don't receive the envProviderState for
// whatever reason, we don't expose the task runner's env to the code
allData.envProviderState ?? {
env: {},
isEnvAccessBlocked: false,
isProcessAvailable: true,
},
).getDataProxy();
}

View file

@ -1,16 +1,17 @@
import {
type IExecuteFunctions,
type Workflow,
type IRunExecutionData,
type INodeExecutionData,
type ITaskDataConnections,
type INode,
type WorkflowParameters,
type INodeParameters,
type WorkflowExecuteMode,
type IExecuteData,
type IDataObject,
type IWorkflowExecuteAdditionalData,
import type {
EnvProviderState,
IExecuteFunctions,
Workflow,
IRunExecutionData,
INodeExecutionData,
ITaskDataConnections,
INode,
WorkflowParameters,
INodeParameters,
WorkflowExecuteMode,
IExecuteData,
IDataObject,
IWorkflowExecuteAdditionalData,
} from 'n8n-workflow';
import { nanoid } from 'nanoid';
@ -42,6 +43,7 @@ export interface TaskData {
connectionInputData: INodeExecutionData[];
siblingParameters: INodeParameters;
mode: WorkflowExecuteMode;
envProviderState: EnvProviderState;
executeData?: IExecuteData;
defaultReturnRunIndex: number;
selfData: IDataObject;
@ -76,6 +78,7 @@ export interface AllCodeTaskData {
connectionInputData: INodeExecutionData[];
siblingParameters: INodeParameters;
mode: WorkflowExecuteMode;
envProviderState: EnvProviderState;
executeData?: IExecuteData;
defaultReturnRunIndex: number;
selfData: IDataObject;
@ -137,6 +140,7 @@ export class TaskManager {
connectionInputData: INodeExecutionData[],
siblingParameters: INodeParameters,
mode: WorkflowExecuteMode,
envProviderState: EnvProviderState,
executeData?: IExecuteData,
defaultReturnRunIndex = -1,
selfData: IDataObject = {},
@ -153,6 +157,7 @@ export class TaskManager {
itemIndex,
siblingParameters,
mode,
envProviderState,
executeData,
defaultReturnRunIndex,
selfData,
@ -311,6 +316,7 @@ export class TaskManager {
contextNodeName: jd.contextNodeName,
defaultReturnRunIndex: jd.defaultReturnRunIndex,
mode: jd.mode,
envProviderState: jd.envProviderState,
node: jd.node,
runExecutionData: jd.runExecutionData,
runIndex: jd.runIndex,

View file

@ -6,6 +6,13 @@
import type { PushType } from '@n8n/api-types';
import { GlobalConfig } from '@n8n/config';
import { WorkflowExecute } from 'n8n-core';
import {
ApplicationError,
ErrorReporterProxy as ErrorReporter,
NodeOperationError,
Workflow,
WorkflowHooks,
} from 'n8n-workflow';
import type {
IDataObject,
IExecuteData,
@ -28,13 +35,7 @@ import type {
ITaskDataConnections,
ExecuteWorkflowOptions,
IWorkflowExecutionDataProcess,
} from 'n8n-workflow';
import {
ApplicationError,
ErrorReporterProxy as ErrorReporter,
NodeOperationError,
Workflow,
WorkflowHooks,
EnvProviderState,
} from 'n8n-workflow';
import { Container } from 'typedi';
@ -1008,6 +1009,7 @@ export async function getBase(
connectionInputData: INodeExecutionData[],
siblingParameters: INodeParameters,
mode: WorkflowExecuteMode,
envProviderState: EnvProviderState,
executeData?: IExecuteData,
defaultReturnRunIndex?: number,
selfData?: IDataObject,
@ -1028,6 +1030,7 @@ export async function getBase(
connectionInputData,
siblingParameters,
mode,
envProviderState,
executeData,
defaultReturnRunIndex,
selfData,

View file

@ -11,6 +11,7 @@ import type {
IExecuteData,
IDataObject,
} from 'n8n-workflow';
import { createEnvProviderState } from 'n8n-workflow';
export const createAgentStartJob = (
additionalData: IWorkflowExecuteAdditionalData,
@ -49,6 +50,7 @@ export const createAgentStartJob = (
connectionInputData,
siblingParameters,
mode,
createEnvProviderState(),
executeData,
defaultReturnRunIndex,
selfData,

View file

@ -22,6 +22,7 @@ import type { WorkflowActivationError } from './errors/workflow-activation.error
import type { WorkflowOperationError } from './errors/workflow-operation.error';
import type { ExecutionStatus } from './ExecutionStatus';
import type { Workflow } from './Workflow';
import type { EnvProviderState } from './WorkflowDataProxyEnvProvider';
import type { WorkflowHooks } from './WorkflowHooks';
export interface IAdditionalCredentialOptions {
@ -2256,6 +2257,7 @@ export interface IWorkflowExecuteAdditionalData {
connectionInputData: INodeExecutionData[],
siblingParameters: INodeParameters,
mode: WorkflowExecuteMode,
envProviderState: EnvProviderState,
executeData?: IExecuteData,
defaultReturnRunIndex?: number,
selfData?: IDataObject,

View file

@ -30,6 +30,8 @@ import {
import * as NodeHelpers from './NodeHelpers';
import { deepCopy } from './utils';
import type { Workflow } from './Workflow';
import type { EnvProviderState } from './WorkflowDataProxyEnvProvider';
import { createEnvProvider, createEnvProviderState } from './WorkflowDataProxyEnvProvider';
import { getPinDataIfManualExecution } from './WorkflowDataProxyHelpers';
export function isResourceLocatorValue(value: unknown): value is INodeParameterResourceLocator {
@ -66,6 +68,7 @@ export class WorkflowDataProxy {
private defaultReturnRunIndex = -1,
private selfData: IDataObject = {},
private contextNodeName: string = activeNodeName,
private envProviderState?: EnvProviderState,
) {
this.runExecutionData = isScriptingNode(this.contextNodeName, workflow)
? runExecutionData !== null
@ -487,40 +490,6 @@ export class WorkflowDataProxy {
);
}
/**
* Returns a proxy to query data from the environment
*
* @private
*/
private envGetter() {
const that = this;
return new Proxy(
{},
{
has: () => true,
get(_, name) {
if (name === 'isProxy') return true;
if (typeof process === 'undefined') {
throw new ExpressionError('not accessible via UI, please run node', {
runIndex: that.runIndex,
itemIndex: that.itemIndex,
});
}
if (process.env.N8N_BLOCK_ENV_ACCESS_IN_NODE === 'true') {
throw new ExpressionError('access to env vars denied', {
causeDetailed:
'If you need access please contact the administrator to remove the environment variable N8N_BLOCK_ENV_ACCESS_IN_NODE',
runIndex: that.runIndex,
itemIndex: that.itemIndex,
});
}
return process.env[name.toString()];
},
},
);
}
private prevNodeGetter() {
const allowedValues = ['name', 'outputIndex', 'runIndex'];
const that = this;
@ -1303,7 +1272,11 @@ export class WorkflowDataProxy {
$binary: {}, // Placeholder
$data: {}, // Placeholder
$env: this.envGetter(),
$env: createEnvProvider(
that.runIndex,
that.itemIndex,
that.envProviderState ?? createEnvProviderState(),
),
$evaluateExpression: (expression: string, itemIndex?: number) => {
itemIndex = itemIndex || that.itemIndex;
return that.workflow.expression.getParameterValue(

View file

@ -0,0 +1,75 @@
import { ExpressionError } from './errors/expression.error';
export type EnvProviderState = {
isProcessAvailable: boolean;
isEnvAccessBlocked: boolean;
env: Record<string, string>;
};
/**
* Captures a snapshot of the environment variables and configuration
* that can be used to initialize an environment provider.
*/
export function createEnvProviderState(): EnvProviderState {
const isProcessAvailable = typeof process !== 'undefined';
const isEnvAccessBlocked = isProcessAvailable
? process.env.N8N_BLOCK_ENV_ACCESS_IN_NODE === 'true'
: false;
const env: Record<string, string> =
!isProcessAvailable || isEnvAccessBlocked ? {} : (process.env as Record<string, string>);
return {
isProcessAvailable,
isEnvAccessBlocked,
env,
};
}
/**
* Creates a proxy that provides access to the environment variables
* in the `WorkflowDataProxy`. Use the `createEnvProviderState` to
* create the default state object that is needed for the proxy,
* unless you need something specific.
*
* @example
* createEnvProvider(
* runIndex,
* itemIndex,
* createEnvProviderState(),
* )
*/
export function createEnvProvider(
runIndex: number,
itemIndex: number,
providerState: EnvProviderState,
): Record<string, string> {
return new Proxy(
{},
{
has() {
return true;
},
get(_, name) {
if (name === 'isProxy') return true;
if (!providerState.isProcessAvailable) {
throw new ExpressionError('not accessible via UI, please run node', {
runIndex,
itemIndex,
});
}
if (providerState.isEnvAccessBlocked) {
throw new ExpressionError('access to env vars denied', {
causeDetailed:
'If you need access please contact the administrator to remove the environment variable N8N_BLOCK_ENV_ACCESS_IN_NODE',
runIndex,
itemIndex,
});
}
return providerState.env[name.toString()];
},
},
);
}

View file

@ -18,6 +18,7 @@ export * from './NodeHelpers';
export * from './RoutingNode';
export * from './Workflow';
export * from './WorkflowDataProxy';
export * from './WorkflowDataProxyEnvProvider';
export * from './WorkflowHooks';
export * from './VersionedNodeType';
export * from './TypeValidation';

View file

@ -0,0 +1,87 @@
import { ExpressionError } from '../src/errors/expression.error';
import { createEnvProvider, createEnvProviderState } from '../src/WorkflowDataProxyEnvProvider';
describe('createEnvProviderState', () => {
afterEach(() => {
delete process.env.N8N_BLOCK_ENV_ACCESS_IN_NODE;
});
it('should return the state with process available and env access allowed', () => {
expect(createEnvProviderState()).toEqual({
isProcessAvailable: true,
isEnvAccessBlocked: false,
env: process.env,
});
});
it('should block env access when N8N_BLOCK_ENV_ACCESS_IN_NODE is set to "true"', () => {
process.env.N8N_BLOCK_ENV_ACCESS_IN_NODE = 'true';
expect(createEnvProviderState()).toEqual({
isProcessAvailable: true,
isEnvAccessBlocked: true,
env: {},
});
});
it('should handle process not being available', () => {
const originalProcess = global.process;
try {
// @ts-expect-error process is read-only
global.process = undefined;
expect(createEnvProviderState()).toEqual({
isProcessAvailable: false,
isEnvAccessBlocked: false,
env: {},
});
} finally {
global.process = originalProcess;
}
});
});
describe('createEnvProvider', () => {
it('should return true when checking for a property using "has"', () => {
const proxy = createEnvProvider(0, 0, createEnvProviderState());
expect('someProperty' in proxy).toBe(true);
});
it('should return the value from process.env if access is allowed', () => {
process.env.TEST_ENV_VAR = 'test_value';
const proxy = createEnvProvider(0, 0, createEnvProviderState());
expect(proxy.TEST_ENV_VAR).toBe('test_value');
});
it('should throw ExpressionError when process is unavailable', () => {
const originalProcess = global.process;
// @ts-expect-error process is read-only
global.process = undefined;
try {
const proxy = createEnvProvider(1, 1, createEnvProviderState());
expect(() => proxy.someEnvVar).toThrowError(
new ExpressionError('not accessible via UI, please run node', {
runIndex: 1,
itemIndex: 1,
}),
);
} finally {
global.process = originalProcess;
}
});
it('should throw ExpressionError when env access is blocked', () => {
process.env.N8N_BLOCK_ENV_ACCESS_IN_NODE = 'true';
const proxy = createEnvProvider(1, 1, createEnvProviderState());
expect(() => proxy.someEnvVar).toThrowError(
new ExpressionError('access to env vars denied', {
causeDetailed:
'If you need access please contact the administrator to remove the environment variable N8N_BLOCK_ENV_ACCESS_IN_NODE',
runIndex: 1,
itemIndex: 1,
}),
);
});
});

View file

@ -641,6 +641,10 @@ importers:
ws:
specifier: '>=8.17.1'
version: 8.17.1
devDependencies:
luxon:
specifier: 'catalog:'
version: 3.4.4
packages/@n8n_io/eslint-config:
devDependencies: