diff --git a/packages/cli/BREAKING-CHANGES.md b/packages/cli/BREAKING-CHANGES.md index 150554886c..03f23cab4d 100644 --- a/packages/cli/BREAKING-CHANGES.md +++ b/packages/cli/BREAKING-CHANGES.md @@ -2,6 +2,49 @@ This list shows all the versions which include breaking changes and how to upgrade. +## 0.135.0 + +### What changed? + +The in-node core methods for credentials and binary data have changed. + +### When is action necessary? + +If you are using custom n8n nodes. + +### How to upgrade: + +1. The method `this.getCredentials(myNodeCredentials)` is now async. So `await` has to be added in front of it. + +Example: + +```typescript +// Before 0.135.0: +const credentials = this.getCredentials(credentialTypeName); + +// From 0.135.0: +const credentials = await this.getCredentials(myNodeCredentials); +``` + +2. Binary data should not get accessed directly anymore, instead the method `await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName)` has to be used. + +Example: + +```typescript + +const items = this.getInputData(); + +for (const i = 0; i < items.length; i++) { + const item = items[i].binary as IBinaryKeyData; + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string; + const binaryData = item[binaryPropertyName] as IBinaryData; + // Before 0.135.0: + const binaryDataBuffer = Buffer.from(binaryData.data, BINARY_ENCODING); + // From 0.135.0: + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); +} +``` + ## 0.131.0 ### What changed? diff --git a/packages/cli/commands/execute.ts b/packages/cli/commands/execute.ts index 48ae6a19f0..662d372023 100644 --- a/packages/cli/commands/execute.ts +++ b/packages/cli/commands/execute.ts @@ -155,10 +155,7 @@ export class Execute extends Command { } try { - const credentials = await WorkflowCredentials(workflowData!.nodes); - const runData: IWorkflowExecutionDataProcess = { - credentials, executionMode: 'cli', startNodes: [startNode.name], workflowData: workflowData!, diff --git a/packages/cli/commands/executeBatch.ts b/packages/cli/commands/executeBatch.ts index dc5f4030db..b1022c6561 100644 --- a/packages/cli/commands/executeBatch.ts +++ b/packages/cli/commands/executeBatch.ts @@ -635,10 +635,8 @@ export class ExecuteBatch extends Command { try { - const credentials = await WorkflowCredentials(workflowData!.nodes); const runData: IWorkflowExecutionDataProcess = { - credentials, executionMode: 'cli', startNodes: [startNode!.name], workflowData: workflowData!, diff --git a/packages/cli/commands/import/credentials.ts b/packages/cli/commands/import/credentials.ts index af2d7e0d46..38835ed7fd 100644 --- a/packages/cli/commands/import/credentials.ts +++ b/packages/cli/commands/import/credentials.ts @@ -21,7 +21,7 @@ import { } from 'n8n-workflow'; import * as fs from 'fs'; -import * as glob from 'glob-promise'; +import * as glob from 'fast-glob'; import * as path from 'path'; export class ImportCredentialsCommand extends Command { diff --git a/packages/cli/commands/import/workflow.ts b/packages/cli/commands/import/workflow.ts index 65ddb77000..2d9a3ac136 100644 --- a/packages/cli/commands/import/workflow.ts +++ b/packages/cli/commands/import/workflow.ts @@ -16,7 +16,7 @@ import { } from 'n8n-workflow'; import * as fs from 'fs'; -import * as glob from 'glob-promise'; +import * as glob from 'fast-glob'; import * as path from 'path'; import { UserSettings, diff --git a/packages/cli/commands/start.ts b/packages/cli/commands/start.ts index bb6e64f56f..d2dbfe5a97 100644 --- a/packages/cli/commands/start.ts +++ b/packages/cli/commands/start.ts @@ -22,10 +22,11 @@ import { NodeTypes, Server, TestWebhooks, + WaitTracker, } from '../src'; import { IDataObject } from 'n8n-workflow'; -import { +import { getLogger, } from '../src/Logger'; @@ -284,6 +285,8 @@ export class Start extends Command { activeWorkflowRunner = ActiveWorkflowRunner.getInstance(); await activeWorkflowRunner.init(); + const waitTracker = WaitTracker(); + const editorUrl = GenericHelpers.getBaseUrl(); this.log(`\nEditor is now accessible via:\n${editorUrl}`); diff --git a/packages/cli/commands/worker.ts b/packages/cli/commands/worker.ts index 56da5d9c98..6e43857d34 100644 --- a/packages/cli/commands/worker.ts +++ b/packages/cli/commands/worker.ts @@ -37,7 +37,7 @@ import { WorkflowExecuteAdditionalData, } from '../src'; -import { +import { getLogger, } from '../src/Logger'; @@ -148,10 +148,9 @@ export class Worker extends Command { const workflow = new Workflow({ id: currentExecutionDb.workflowData.id as string, name: currentExecutionDb.workflowData.name, nodes: currentExecutionDb.workflowData!.nodes, connections: currentExecutionDb.workflowData!.connections, active: currentExecutionDb.workflowData!.active, nodeTypes, staticData, settings: currentExecutionDb.workflowData!.settings }); - const credentials = await WorkflowCredentials(currentExecutionDb.workflowData.nodes); - - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials, undefined, executionTimeoutTimestamp); + const additionalData = await WorkflowExecuteAdditionalData.getBase(undefined, executionTimeoutTimestamp); additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerExecuter(currentExecutionDb.mode, job.data.executionId, currentExecutionDb.workflowData, { retryOf: currentExecutionDb.retryOf as string }); + additionalData.executionId = jobData.executionId; let workflowExecute: WorkflowExecute; let workflowRun: PCancelable; diff --git a/packages/cli/config/index.ts b/packages/cli/config/index.ts index 476cbbcfa7..0a5273799b 100644 --- a/packages/cli/config/index.ts +++ b/packages/cli/config/index.ts @@ -489,6 +489,12 @@ const config = convict({ env: 'N8N_ENDPOINT_WEBHOOK', doc: 'Path for webhook endpoint', }, + webhookWaiting: { + format: String, + default: 'webhook-waiting', + env: 'N8N_ENDPOINT_WEBHOOK_WAIT', + doc: 'Path for waiting-webhook endpoint', + }, webhookTest: { format: String, default: 'webhook-test', diff --git a/packages/cli/package.json b/packages/cli/package.json index 6fb974fdc9..3437ae85d1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.134.0", + "version": "0.135.1", "description": "n8n Workflow Automation Tool", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", @@ -101,8 +101,8 @@ "csrf": "^3.1.0", "dotenv": "^8.0.0", "express": "^4.16.4", + "fast-glob": "^3.2.5", "flatted": "^2.0.0", - "glob-promise": "^3.4.0", "google-timezones-json": "^1.0.2", "inquirer": "^7.0.1", "json-diff": "^0.5.4", @@ -111,10 +111,10 @@ "localtunnel": "^2.0.0", "lodash.get": "^4.4.2", "mysql2": "~2.2.0", - "n8n-core": "~0.79.0", - "n8n-editor-ui": "~0.102.0", - "n8n-nodes-base": "~0.131.0", - "n8n-workflow": "~0.64.0", + "n8n-core": "~0.80.0", + "n8n-editor-ui": "~0.103.0", + "n8n-nodes-base": "~0.132.0", + "n8n-workflow": "~0.65.0", "oauth-1.0a": "^2.2.6", "open": "^7.0.0", "pg": "^8.3.0", diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts index b2a9fb0a06..3741ccd19e 100644 --- a/packages/cli/src/ActiveExecutions.ts +++ b/packages/cli/src/ActiveExecutions.ts @@ -35,32 +35,43 @@ export class ActiveExecutions { * @returns {string} * @memberof ActiveExecutions */ - async add(executionData: IWorkflowExecutionDataProcess, process?: ChildProcess): Promise { + async add(executionData: IWorkflowExecutionDataProcess, process?: ChildProcess, executionId?: string): Promise { - const fullExecutionData: IExecutionDb = { - data: executionData.executionData!, - mode: executionData.executionMode, - finished: false, - startedAt: new Date(), - workflowData: executionData.workflowData, - }; + if (executionId === undefined) { + // Is a new execution so save in DB - if (executionData.retryOf !== undefined) { - fullExecutionData.retryOf = executionData.retryOf.toString(); + const fullExecutionData: IExecutionDb = { + data: executionData.executionData!, + mode: executionData.executionMode, + finished: false, + startedAt: new Date(), + workflowData: executionData.workflowData, + }; + + if (executionData.retryOf !== undefined) { + fullExecutionData.retryOf = executionData.retryOf.toString(); + } + + if (executionData.workflowData.id !== undefined && WorkflowHelpers.isWorkflowIdValid(executionData.workflowData.id.toString()) === true) { + fullExecutionData.workflowId = executionData.workflowData.id.toString(); + } + + const execution = ResponseHelper.flattenExecutionData(fullExecutionData); + + const executionResult = await Db.collections.Execution!.save(execution as IExecutionFlattedDb); + executionId = typeof executionResult.id === "object" ? executionResult.id!.toString() : executionResult.id + ""; + } else { + // Is an existing execution we want to finish so update in DB + + const execution = { + id: executionId, + waitTill: null, + }; + + // @ts-ignore + await Db.collections.Execution!.update(executionId, execution); } - if (executionData.workflowData.id !== undefined && WorkflowHelpers.isWorkflowIdValid(executionData.workflowData.id.toString()) === true) { - fullExecutionData.workflowId = executionData.workflowData.id.toString(); - } - - const execution = ResponseHelper.flattenExecutionData(fullExecutionData); - - // Save the Execution in DB - const executionResult = await Db.collections.Execution!.save(execution as IExecutionFlattedDb); - - // @ts-ignore - const executionId = typeof executionResult.id === "object" ? executionResult.id!.toString() : executionResult.id + ""; - this.activeExecutions[executionId] = { executionData, process, diff --git a/packages/cli/src/ActiveWorkflowRunner.ts b/packages/cli/src/ActiveWorkflowRunner.ts index 80394bd136..02d0b27691 100644 --- a/packages/cli/src/ActiveWorkflowRunner.ts +++ b/packages/cli/src/ActiveWorkflowRunner.ts @@ -194,9 +194,7 @@ export class ActiveWorkflowRunner { const nodeTypes = NodeTypes(); const workflow = new Workflow({ id: webhook.workflowId.toString(), name: workflowData.name, nodes: workflowData.nodes, connections: workflowData.connections, active: workflowData.active, nodeTypes, staticData: workflowData.staticData, settings: workflowData.settings }); - const credentials = await WorkflowCredentials([workflow.getNode(webhook.node as string) as INode]); - - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials); + const additionalData = await WorkflowExecuteAdditionalData.getBase(); const webhookData = NodeHelpers.getNodeWebhooks(workflow, workflow.getNode(webhook.node as string) as INode, additionalData).filter((webhook) => { return (webhook.httpMethod === httpMethod && webhook.path === path); @@ -213,7 +211,7 @@ export class ActiveWorkflowRunner { return new Promise((resolve, reject) => { const executionMode = 'webhook'; //@ts-ignore - WebhookHelpers.executeWebhook(workflow, webhookData, workflowData, workflowStartNode, executionMode, undefined, req, res, (error: Error | null, data: object) => { + WebhookHelpers.executeWebhook(workflow, webhookData, workflowData, workflowStartNode, executionMode, undefined, undefined, undefined, req, res, (error: Error | null, data: object) => { if (error !== null) { return reject(error); } @@ -286,7 +284,7 @@ export class ActiveWorkflowRunner { * @memberof ActiveWorkflowRunner */ async addWorkflowWebhooks(workflow: Workflow, additionalData: IWorkflowExecuteAdditionalDataWorkflow, mode: WorkflowExecuteMode, activation: WorkflowActivateMode): Promise { - const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData); + const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData, undefined, true); let path = '' as string | undefined; for (const webhookData of webhooks) { @@ -370,10 +368,9 @@ export class ActiveWorkflowRunner { const mode = 'internal'; - const credentials = await WorkflowCredentials(workflowData.nodes); - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials); + const additionalData = await WorkflowExecuteAdditionalData.getBase(); - const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData); + const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData, undefined, true); for (const webhookData of webhooks) { await workflow.runWebhookMethod('delete', webhookData, NodeExecuteFunctions, mode, 'update', false); @@ -423,7 +420,6 @@ export class ActiveWorkflowRunner { // Start the workflow const runData: IWorkflowExecutionDataProcess = { - credentials: additionalData.credentials, executionMode: mode, executionData, workflowData, @@ -510,8 +506,7 @@ export class ActiveWorkflowRunner { } const mode = 'trigger'; - const credentials = await WorkflowCredentials(workflowData.nodes); - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials); + const additionalData = await WorkflowExecuteAdditionalData.getBase(); const getTriggerFunctions = this.getExecuteTriggerFunctions(workflowData, additionalData, mode, activation); const getPollFunctions = this.getExecutePollFunctions(workflowData, additionalData, mode, activation); diff --git a/packages/cli/src/CredentialsHelper.ts b/packages/cli/src/CredentialsHelper.ts index 07e9f484f7..fab915f73e 100644 --- a/packages/cli/src/CredentialsHelper.ts +++ b/packages/cli/src/CredentialsHelper.ts @@ -48,16 +48,21 @@ export class CredentialsHelper extends ICredentialsHelper { * @returns {Credentials} * @memberof CredentialsHelper */ - getCredentials(name: string, type: string): Credentials { - if (!this.workflowCredentials[type]) { + async getCredentials(name: string, type: string): Promise { + + const credentialsDb = await Db.collections.Credentials?.find({type}); + + if (credentialsDb === undefined || credentialsDb.length === 0) { throw new Error(`No credentials of type "${type}" exist.`); } - if (!this.workflowCredentials[type][name]) { + + const credential = credentialsDb.find(credential => credential.name === name); + + if (credential === undefined) { throw new Error(`No credentials with name "${name}" exist for type "${type}".`); } - const credentialData = this.workflowCredentials[type][name]; - - return new Credentials(credentialData.name, credentialData.type, credentialData.nodesAccess, credentialData.data); + + return new Credentials(credential.name, credential.type, credential.nodesAccess, credential.data); } @@ -102,8 +107,8 @@ export class CredentialsHelper extends ICredentialsHelper { * @returns {ICredentialDataDecryptedObject} * @memberof CredentialsHelper */ - getDecrypted(name: string, type: string, mode: WorkflowExecuteMode, raw?: boolean, expressionResolveValues?: ICredentialsExpressionResolveValues): ICredentialDataDecryptedObject { - const credentials = this.getCredentials(name, type); + async getDecrypted(name: string, type: string, mode: WorkflowExecuteMode, raw?: boolean, expressionResolveValues?: ICredentialsExpressionResolveValues): Promise { + const credentials = await this.getCredentials(name, type); const decryptedDataOriginal = credentials.getData(this.encryptionKey); @@ -138,7 +143,7 @@ export class CredentialsHelper extends ICredentialsHelper { if (expressionResolveValues) { try { const workflow = new Workflow({ nodes: Object.values(expressionResolveValues.workflow.nodes), connections: expressionResolveValues.workflow.connectionsBySourceNode, active: false, nodeTypes: expressionResolveValues.workflow.nodeTypes }); - decryptedData = workflow.expression.getParameterValue(decryptedData as INodeParameters, expressionResolveValues.runExecutionData, expressionResolveValues.runIndex, expressionResolveValues.itemIndex, expressionResolveValues.node.name, expressionResolveValues.connectionInputData, mode, false, decryptedData) as ICredentialDataDecryptedObject; + decryptedData = workflow.expression.getParameterValue(decryptedData as INodeParameters, expressionResolveValues.runExecutionData, expressionResolveValues.runIndex, expressionResolveValues.itemIndex, expressionResolveValues.node.name, expressionResolveValues.connectionInputData, mode, {}, false, decryptedData) as ICredentialDataDecryptedObject; } catch (e) { e.message += ' [Error resolving credentials]'; throw e; @@ -155,7 +160,7 @@ export class CredentialsHelper extends ICredentialsHelper { const workflow = new Workflow({ nodes: [node!], connections: {}, active: false, nodeTypes: mockNodeTypes }); // Resolve expressions if any are set - decryptedData = workflow.expression.getComplexParameterValue(node!, decryptedData as INodeParameters, mode, undefined, decryptedData) as ICredentialDataDecryptedObject; + decryptedData = workflow.expression.getComplexParameterValue(node!, decryptedData as INodeParameters, mode, {}, undefined, decryptedData) as ICredentialDataDecryptedObject; } // Load and apply the credentials overwrites if any exist diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index d2aff0eb86..5ee6a48b7e 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -150,6 +150,7 @@ export interface IExecutionBase { // Data in regular format with references export interface IExecutionDb extends IExecutionBase { data: IRunExecutionData; + waitTill?: Date; workflowData?: IWorkflowBase; } @@ -163,6 +164,7 @@ export interface IExecutionResponse extends IExecutionBase { data: IRunExecutionData; retryOf?: string; retrySuccessId?: string; + waitTill?: Date; workflowData: IWorkflowBase; } @@ -176,6 +178,7 @@ export interface IExecutionFlatted extends IExecutionBase { export interface IExecutionFlattedDb extends IExecutionBase { id: number | string; data: string; + waitTill?: Date | null; workflowData: IWorkflowBase; } @@ -204,6 +207,7 @@ export interface IExecutionsSummary { mode: WorkflowExecuteMode; retryOf?: string; retrySuccessId?: string; + waitTill?: Date; startedAt: Date; stoppedAt?: Date; workflowId: string; @@ -457,7 +461,6 @@ export interface IProcessMessageDataHook { } export interface IWorkflowExecutionDataProcess { - credentials: IWorkflowCredentials; destinationNode?: string; executionMode: WorkflowExecuteMode; executionData?: IRunExecutionData; diff --git a/packages/cli/src/LoadNodesAndCredentials.ts b/packages/cli/src/LoadNodesAndCredentials.ts index 777fa0bb00..037293dfc2 100644 --- a/packages/cli/src/LoadNodesAndCredentials.ts +++ b/packages/cli/src/LoadNodesAndCredentials.ts @@ -22,8 +22,8 @@ import { readdir as fsReaddir, readFile as fsReadFile, stat as fsStat, - } from 'fs/promises'; -import * as glob from 'glob-promise'; +} from 'fs/promises'; +import * as glob from 'fast-glob'; import * as path from 'path'; const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; diff --git a/packages/cli/src/ResponseHelper.ts b/packages/cli/src/ResponseHelper.ts index 24a9d37b53..bb447a91ba 100644 --- a/packages/cli/src/ResponseHelper.ts +++ b/packages/cli/src/ResponseHelper.ts @@ -163,6 +163,7 @@ export function flattenExecutionData(fullExecutionData: IExecutionDb): IExecutio const returnData: IExecutionFlatted = Object.assign({}, { data: stringify(fullExecutionData.data), mode: fullExecutionData.mode, + waitTill: fullExecutionData.waitTill, startedAt: fullExecutionData.startedAt, stoppedAt: fullExecutionData.stoppedAt, finished: fullExecutionData.finished ? fullExecutionData.finished : false, @@ -200,6 +201,7 @@ export function unflattenExecutionData(fullExecutionData: IExecutionFlattedDb): workflowData: fullExecutionData.workflowData as IWorkflowDb, data: parse(fullExecutionData.data), mode: fullExecutionData.mode, + waitTill: fullExecutionData.waitTill ? fullExecutionData.waitTill : undefined, startedAt: fullExecutionData.startedAt, stoppedAt: fullExecutionData.stoppedAt, finished: fullExecutionData.finished ? fullExecutionData.finished : false, diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index d4341ed711..f180c4b469 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -64,9 +64,11 @@ import { Push, ResponseHelper, TestWebhooks, + WaitingWebhooks, + WaitTracker, + WaitTrackerClass, WebhookHelpers, WebhookServer, - WorkflowCredentials, WorkflowExecuteAdditionalData, WorkflowHelpers, WorkflowRunner, @@ -97,6 +99,7 @@ import { import { FindManyOptions, FindOneOptions, + IsNull, LessThanOrEqual, Not, } from 'typeorm'; @@ -125,9 +128,11 @@ class App { activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner; testWebhooks: TestWebhooks.TestWebhooks; endpointWebhook: string; + endpointWebhookWaiting: string; endpointWebhookTest: string; endpointPresetCredentials: string; externalHooks: IExternalHooksClass; + waitTracker: WaitTrackerClass; defaultWorkflowName: string; saveDataErrorExecution: string; saveDataSuccessExecution: string; @@ -151,6 +156,7 @@ class App { this.app = express(); this.endpointWebhook = config.get('endpoints.webhook') as string; + this.endpointWebhookWaiting = config.get('endpoints.webhookWaiting') as string; this.endpointWebhookTest = config.get('endpoints.webhookTest') as string; this.defaultWorkflowName = config.get('workflows.defaultName') as string; @@ -169,6 +175,7 @@ class App { this.push = Push.getInstance(); this.activeExecutionsInstance = ActiveExecutions.getInstance(); + this.waitTracker = WaitTracker(); this.protocol = config.get('protocol'); this.sslKey = config.get('ssl_key'); @@ -621,7 +628,6 @@ class App { return { name: `${nameToReturn} ${maxSuffix + 1}` }; })); - // Returns a specific workflow this.app.get(`/${this.restEndpoint}/workflows/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { const workflow = await Db.collections.Workflow!.findOne(req.params.id, { relations: ['tags'] }); @@ -764,8 +770,7 @@ class App { // If webhooks nodes exist and are active we have to wait for till we receive a call if (runData === undefined || startNodes === undefined || startNodes.length === 0 || destinationNode === undefined) { - const credentials = await WorkflowCredentials(workflowData.nodes); - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials); + const additionalData = await WorkflowExecuteAdditionalData.getBase(); const nodeTypes = NodeTypes(); const workflowInstance = new Workflow({ id: workflowData.id, name: workflowData.name, nodes: workflowData.nodes, connections: workflowData.connections, active: false, nodeTypes, staticData: undefined, settings: workflowData.settings }); const needsWebhook = await this.testWebhooks.needsWebhookData(workflowData, workflowInstance, additionalData, executionMode, activationMode, sessionId, destinationNode); @@ -779,11 +784,8 @@ class App { // For manual testing always set to not active workflowData.active = false; - const credentials = await WorkflowCredentials(workflowData.nodes); - // Start the workflow const data: IWorkflowExecutionDataProcess = { - credentials, destinationNode, executionMode, runData, @@ -880,9 +882,7 @@ class App { // @ts-ignore const loadDataInstance = new LoadNodeParameterOptions(nodeType, nodeTypes, path, JSON.parse('' + req.query.currentNodeParameters), credentials!); - const workflowData = loadDataInstance.getWorkflowData() as IWorkflowBase; - const workflowCredentials = await WorkflowCredentials(workflowData.nodes); - const additionalData = await WorkflowExecuteAdditionalData.getBase(workflowCredentials, currentNodeParameters); + const additionalData = await WorkflowExecuteAdditionalData.getBase(currentNodeParameters); return loadDataInstance.getOptions(methodName, additionalData); })); @@ -1259,15 +1259,9 @@ class App { return ''; } - // Decrypt the currently saved credentials - const workflowCredentials: IWorkflowCredentials = { - [result.type as string]: { - [result.name as string]: result as ICredentialsEncrypted, - }, - }; const mode: WorkflowExecuteMode = 'internal'; - const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey); - const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true); + const credentialsHelper = new CredentialsHelper(encryptionKey); + const decryptedDataOriginal = await credentialsHelper.getDecrypted(result.name, result.type, mode, true); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode); const signatureMethod = _.get(oauthCredentials, 'signatureMethod') as string; @@ -1351,6 +1345,7 @@ class App { return ResponseHelper.sendErrorResponse(res, errorResponse); } + // Decrypt the currently saved credentials const workflowCredentials: IWorkflowCredentials = { [result.type as string]: { @@ -1358,10 +1353,10 @@ class App { }, }; const mode: WorkflowExecuteMode = 'internal'; - const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey); - const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true); + const credentialsHelper = new CredentialsHelper(encryptionKey); + const decryptedDataOriginal = await credentialsHelper.getDecrypted(result.name, result.type, mode, true); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode); - + const options: OptionsWithUrl = { method: 'POST', url: _.get(oauthCredentials, 'accessTokenUrl') as string, @@ -1427,15 +1422,9 @@ class App { return ''; } - // Decrypt the currently saved credentials - const workflowCredentials: IWorkflowCredentials = { - [result.type as string]: { - [result.name as string]: result as ICredentialsEncrypted, - }, - }; const mode: WorkflowExecuteMode = 'internal'; - const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey); - const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true); + const credentialsHelper = new CredentialsHelper(encryptionKey); + const decryptedDataOriginal = await credentialsHelper.getDecrypted(result.name, result.type, mode, true); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode); const token = new csrf(); @@ -1534,11 +1523,12 @@ class App { [result.name as string]: result as ICredentialsEncrypted, }, }; + const mode: WorkflowExecuteMode = 'internal'; - const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey); - const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true); + const credentialsHelper = new CredentialsHelper(encryptionKey); + const decryptedDataOriginal = await credentialsHelper.getDecrypted(result.name, result.type, mode, true); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode); - + const token = new csrf(); if (decryptedDataOriginal.csrfSecret === undefined || !token.verify(decryptedDataOriginal.csrfSecret as string, state.token)) { const errorResponse = new ResponseHelper.ResponseError('The OAuth2 callback state is invalid!', undefined, 404); @@ -1638,6 +1628,9 @@ class App { executingWorkflowIds.push(...this.activeExecutionsInstance.getActiveExecutions().map(execution => execution.id.toString()) as string[]); const countFilter = JSON.parse(JSON.stringify(filter)); + if (countFilter.waitTill !== undefined) { + countFilter.waitTill = Not(IsNull()); + } countFilter.id = Not(In(executingWorkflowIds)); const resultsQuery = await Db.collections.Execution! @@ -1648,6 +1641,7 @@ class App { 'execution.mode', 'execution.retryOf', 'execution.retrySuccessId', + 'execution.waitTill', 'execution.startedAt', 'execution.stoppedAt', 'execution.workflowData', @@ -1656,7 +1650,14 @@ class App { .take(limit); Object.keys(filter).forEach((filterField) => { - resultsQuery.andWhere(`execution.${filterField} = :${filterField}`, {[filterField]: filter[filterField]}); + if (filterField === 'waitTill') { + resultsQuery.andWhere(`execution.${filterField} is not null`); + } else if(filterField === 'finished' && filter[filterField] === false) { + resultsQuery.andWhere(`execution.${filterField} = :${filterField}`, {[filterField]: filter[filterField]}); + resultsQuery.andWhere(`execution.waitTill is null`); + } else { + resultsQuery.andWhere(`execution.${filterField} = :${filterField}`, {[filterField]: filter[filterField]}); + } }); if (req.query.lastId) { resultsQuery.andWhere(`execution.id < :lastId`, {lastId: req.query.lastId}); @@ -1684,6 +1685,7 @@ class App { mode: result.mode, retryOf: result.retryOf ? result.retryOf.toString() : undefined, retrySuccessId: result.retrySuccessId ? result.retrySuccessId.toString() : undefined, + waitTill: result.waitTill as Date | undefined, startedAt: result.startedAt, stoppedAt: result.stoppedAt, workflowId: result.workflowData!.id ? result.workflowData!.id!.toString() : '', @@ -1735,13 +1737,10 @@ class App { const executionMode = 'retry'; - const credentials = await WorkflowCredentials(fullExecutionData.workflowData.nodes); - fullExecutionData.workflowData.active = false; // Start the workflow const data: IWorkflowExecutionDataProcess = { - credentials, executionMode, executionData: fullExecutionData.data, retryOf: req.params.id, @@ -1913,15 +1912,22 @@ class App { // Manual executions should still be stoppable, so // try notifying the `activeExecutions` to stop it. const result = await this.activeExecutionsInstance.stopExecution(req.params.id); - if (result !== undefined) { - const returnData: IExecutionsStopData = { + + if (result === undefined) { + // If active execution could not be found check if it is a waiting one + try { + return await this.waitTracker.stopExecution(req.params.id); + } catch (error) { + // Ignore, if it errors as then it is probably a currently running + // execution + } + } else { + return { mode: result.mode, startedAt: new Date(result.startedAt), - stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined, + stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined, finished: result.finished, - }; - - return returnData; + } as IExecutionsStopData; } const currentJobs = await Queue.getInstance().getJobs(['active', 'waiting']); @@ -1952,17 +1958,19 @@ class App { // Stopt he execution and wait till it is done and we got the data const result = await this.activeExecutionsInstance.stopExecution(executionId); + let returnData: IExecutionsStopData; if (result === undefined) { - throw new Error(`The execution id "${executionId}" could not be found.`); + // If active execution could not be found check if it is a waiting one + returnData = await this.waitTracker.stopExecution(executionId); + } else { + returnData = { + mode: result.mode, + startedAt: new Date(result.startedAt), + stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined, + finished: result.finished, + }; } - const returnData: IExecutionsStopData = { - mode: result.mode, - startedAt: new Date(result.startedAt), - stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined, - finished: result.finished, - }; - return returnData; } })); @@ -2008,6 +2016,76 @@ class App { WebhookServer.registerProductionWebhooks.apply(this); } + // ---------------------------------------- + // Waiting Webhooks + // ---------------------------------------- + + const waitingWebhooks = new WaitingWebhooks(); + + // HEAD webhook-waiting requests + this.app.head(`/${this.endpointWebhookWaiting}/*`, async (req: express.Request, res: express.Response) => { + // Cut away the "/webhook-waiting/" to get the registred part of the url + const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookWaiting.length + 2); + + let response; + try { + response = await waitingWebhooks.executeWebhook('HEAD', requestUrl, req, res); + } catch (error) { + ResponseHelper.sendErrorResponse(res, error); + return; + } + + if (response.noWebhookResponse === true) { + // Nothing else to do as the response got already sent + return; + } + + ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode); + }); + + // GET webhook-waiting requests + this.app.get(`/${this.endpointWebhookWaiting}/*`, async (req: express.Request, res: express.Response) => { + // Cut away the "/webhook-waiting/" to get the registred part of the url + const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookWaiting.length + 2); + + let response; + try { + response = await waitingWebhooks.executeWebhook('GET', requestUrl, req, res); + } catch (error) { + ResponseHelper.sendErrorResponse(res, error); + return; + } + + if (response.noWebhookResponse === true) { + // Nothing else to do as the response got already sent + return; + } + + ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode); + }); + + // POST webhook-waiting requests + this.app.post(`/${this.endpointWebhookWaiting}/*`, async (req: express.Request, res: express.Response) => { + // Cut away the "/webhook-waiting/" to get the registred part of the url + const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookWaiting.length + 2); + + let response; + try { + response = await waitingWebhooks.executeWebhook('POST', requestUrl, req, res); + } catch (error) { + ResponseHelper.sendErrorResponse(res, error); + return; + } + + if (response.noWebhookResponse === true) { + // Nothing else to do as the response got already sent + return; + } + + ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode); + }); + + // HEAD webhook requests (test for UI) this.app.head(`/${this.endpointWebhookTest}/*`, async (req: express.Request, res: express.Response) => { // Cut away the "/webhook-test/" to get the registred part of the url diff --git a/packages/cli/src/TestWebhooks.ts b/packages/cli/src/TestWebhooks.ts index a8aa17720f..96e6f299a5 100644 --- a/packages/cli/src/TestWebhooks.ts +++ b/packages/cli/src/TestWebhooks.ts @@ -105,7 +105,7 @@ export class TestWebhooks { return new Promise(async (resolve, reject) => { try { const executionMode = 'manual'; - const executionId = await WebhookHelpers.executeWebhook(workflow, webhookData!, this.testWebhookData[webhookKey].workflowData, workflowStartNode, executionMode, this.testWebhookData[webhookKey].sessionId, request, response, (error: Error | null, data: IResponseCallbackData) => { + const executionId = await WebhookHelpers.executeWebhook(workflow, webhookData!, this.testWebhookData[webhookKey].workflowData, workflowStartNode, executionMode, this.testWebhookData[webhookKey].sessionId, undefined, undefined, request, response, (error: Error | null, data: IResponseCallbackData) => { if (error !== null) { return reject(error); } @@ -163,10 +163,9 @@ export class TestWebhooks { * @memberof TestWebhooks */ async needsWebhookData(workflowData: IWorkflowDb, workflow: Workflow, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, activation: WorkflowActivateMode, sessionId?: string, destinationNode?: string): Promise { - const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData, destinationNode); - - if (webhooks.length === 0) { - // No Webhooks found + const webhooks = WebhookHelpers.getWorkflowWebhooks(workflow, additionalData, destinationNode, true); + if (!webhooks.find(webhook => webhook.webhookDescription.restartWebhook !== true)) { + // No webhooks found to start a workflow return false; } diff --git a/packages/cli/src/WaitTracker.ts b/packages/cli/src/WaitTracker.ts new file mode 100644 index 0000000000..81ee39e418 --- /dev/null +++ b/packages/cli/src/WaitTracker.ts @@ -0,0 +1,181 @@ +import { + ActiveExecutions, + DatabaseType, + Db, + GenericHelpers, + IExecutionFlattedDb, + IExecutionsStopData, + IWorkflowExecutionDataProcess, + ResponseHelper, + WorkflowCredentials, + WorkflowRunner, +} from '.'; + +import { + IRun, + LoggerProxy as Logger, + WorkflowOperationError, +} from 'n8n-workflow'; + +import { + FindManyOptions, + LessThanOrEqual, + ObjectLiteral, +} from 'typeorm'; + +import { DateUtils } from 'typeorm/util/DateUtils'; + + +export class WaitTrackerClass { + activeExecutionsInstance: ActiveExecutions.ActiveExecutions; + + private waitingExecutions: { + [key: string]: { + executionId: string, + timer: NodeJS.Timeout, + }; + } = {}; + + mainTimer: NodeJS.Timeout; + + + constructor() { + this.activeExecutionsInstance = ActiveExecutions.getInstance(); + + // Poll every 60 seconds a list of upcoming executions + this.mainTimer = setInterval(() => { + this.getwaitingExecutions(); + }, 60000); + + this.getwaitingExecutions(); + } + + + async getwaitingExecutions() { + Logger.debug('Wait tracker querying database for waiting executions'); + // Find all the executions which should be triggered in the next 70 seconds + const findQuery: FindManyOptions = { + select: ['id', 'waitTill'], + where: { + waitTill: LessThanOrEqual(new Date(Date.now() + 70000)), + }, + order: { + waitTill: 'ASC', + }, + }; + const dbType = await GenericHelpers.getConfigValue('database.type') as DatabaseType; + if (dbType === 'sqlite') { + // This is needed because of issue in TypeORM <> SQLite: + // https://github.com/typeorm/typeorm/issues/2286 + (findQuery.where! as ObjectLiteral).waitTill = LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(new Date(Date.now() + 70000))); + } + + const executions = await Db.collections.Execution!.find(findQuery); + + if (executions.length === 0) { + return; + } + + const executionIds = executions.map(execution => execution.id.toString()).join(', '); + Logger.debug(`Wait tracker found ${executions.length} executions. Setting timer for IDs: ${executionIds}`); + + // Add timers for each waiting execution that they get started at the correct time + for (const execution of executions) { + const executionId = execution.id.toString(); + if (this.waitingExecutions[executionId] === undefined) { + const triggerTime = execution.waitTill!.getTime() - new Date().getTime(); + this.waitingExecutions[executionId] = { + executionId, + timer: setTimeout(() => { + this.startExecution(executionId); + }, triggerTime), + }; + } + } + } + + + async stopExecution(executionId: string): Promise { + if (this.waitingExecutions[executionId] !== undefined) { + // The waiting execution was already sheduled to execute. + // So stop timer and remove. + clearTimeout(this.waitingExecutions[executionId].timer); + delete this.waitingExecutions[executionId]; + } + + // Also check in database + const execution = await Db.collections.Execution!.findOne(executionId); + + if (execution === undefined || !execution.waitTill) { + throw new Error(`The execution ID "${executionId}" could not be found.`); + } + + const fullExecutionData = ResponseHelper.unflattenExecutionData(execution); + + // Set in execution in DB as failed and remove waitTill time + const error = new WorkflowOperationError('Workflow-Execution has been canceled!'); + + fullExecutionData.data.resultData.error = { + ...error, + message: error.message, + stack: error.stack, + }; + + fullExecutionData.stoppedAt = new Date(); + fullExecutionData.waitTill = undefined; + + await Db.collections.Execution!.update(executionId, ResponseHelper.flattenExecutionData(fullExecutionData)); + + return { + mode: fullExecutionData.mode, + startedAt: new Date(fullExecutionData.startedAt), + stoppedAt: fullExecutionData.stoppedAt ? new Date(fullExecutionData.stoppedAt) : undefined, + finished: fullExecutionData.finished, + }; + } + + + startExecution(executionId: string) { + Logger.debug(`Wait tracker resuming execution ${executionId}`, {executionId}); + delete this.waitingExecutions[executionId]; + + (async () => { + // Get the data to execute + const fullExecutionDataFlatted = await Db.collections.Execution!.findOne(executionId); + + if (fullExecutionDataFlatted === undefined) { + throw new Error(`The execution with the id "${executionId}" does not exist.`); + } + + const fullExecutionData = ResponseHelper.unflattenExecutionData(fullExecutionDataFlatted); + + if (fullExecutionData.finished === true) { + throw new Error('The execution did succeed and can so not be started again.'); + } + + const data: IWorkflowExecutionDataProcess = { + executionMode: fullExecutionData.mode, + executionData: fullExecutionData.data, + workflowData: fullExecutionData.workflowData, + }; + + // Start the execution again + const workflowRunner = new WorkflowRunner(); + await workflowRunner.run(data, false, false, executionId); + })().catch((error) => { + Logger.error(`There was a problem starting the waiting execution with id "${executionId}": "${error.message}"`, { executionId }); + }); + + } +} + + +let waitTrackerInstance: WaitTrackerClass | undefined; + +export function WaitTracker(): WaitTrackerClass { + if (waitTrackerInstance === undefined) { + waitTrackerInstance = new WaitTrackerClass(); + } + + return waitTrackerInstance; +} diff --git a/packages/cli/src/WaitingWebhooks.ts b/packages/cli/src/WaitingWebhooks.ts new file mode 100644 index 0000000000..f0b84d3804 --- /dev/null +++ b/packages/cli/src/WaitingWebhooks.ts @@ -0,0 +1,117 @@ +import { + Db, + IExecutionResponse, + IResponseCallbackData, + IWorkflowDb, + NodeTypes, + ResponseHelper, + WebhookHelpers, + WorkflowCredentials, + WorkflowExecuteAdditionalData, +} from '.'; + +import { + INode, + IRunExecutionData, + NodeHelpers, + WebhookHttpMethod, + Workflow, +} from 'n8n-workflow'; + +import * as express from 'express'; +import { + LoggerProxy as Logger, +} from 'n8n-workflow'; + +export class WaitingWebhooks { + + async executeWebhook(httpMethod: WebhookHttpMethod, fullPath: string, req: express.Request, res: express.Response): Promise { + Logger.debug(`Received waiting-webhoook "${httpMethod}" for path "${fullPath}"`); + + // Reset request parameters + req.params = {}; + + // Remove trailing slash + if (fullPath.endsWith('/')) { + fullPath = fullPath.slice(0, -1); + } + + const pathParts = fullPath.split('/'); + + const executionId = pathParts.shift(); + const path = pathParts.join('/'); + + const execution = await Db.collections.Execution?.findOne(executionId); + + if (execution === undefined) { + throw new ResponseHelper.ResponseError(`The execution "${executionId} does not exist.`, 404, 404); + } + + const fullExecutionData = ResponseHelper.unflattenExecutionData(execution); + + if (fullExecutionData.finished === true || fullExecutionData.data.resultData.error) { + throw new ResponseHelper.ResponseError(`The execution "${executionId} has finished already.`, 409, 409); + } + + return this.startExecution(httpMethod, path, fullExecutionData, req, res); + } + + + async startExecution(httpMethod: WebhookHttpMethod, path: string, fullExecutionData: IExecutionResponse, req: express.Request, res: express.Response): Promise { + const executionId = fullExecutionData.id; + + if (fullExecutionData.finished === true) { + throw new Error('The execution did succeed and can so not be started again.'); + } + + const lastNodeExecuted = fullExecutionData!.data.resultData.lastNodeExecuted as string; + + // Set the node as disabled so that the data does not get executed again as it would result + // in starting the wait all over again + fullExecutionData!.data.executionData!.nodeExecutionStack[0].node.disabled = true; + + // Remove waitTill information else the execution would stop + fullExecutionData!.data.waitTill = undefined; + + // Remove the data of the node execution again else it will display the node as executed twice + fullExecutionData!.data.resultData.runData[lastNodeExecuted].pop(); + + const workflowData = fullExecutionData.workflowData; + + const nodeTypes = NodeTypes(); + const workflow = new Workflow({ id: workflowData.id!.toString(), name: workflowData.name, nodes: workflowData.nodes, connections: workflowData.connections, active: workflowData.active, nodeTypes, staticData: workflowData.staticData, settings: workflowData.settings }); + + const additionalData = await WorkflowExecuteAdditionalData.getBase(); + + const webhookData = NodeHelpers.getNodeWebhooks(workflow, workflow.getNode(lastNodeExecuted) as INode, additionalData).filter((webhook) => { + return (webhook.httpMethod === httpMethod && webhook.path === path && webhook.webhookDescription.restartWebhook === true); + })[0]; + + if (webhookData === undefined) { + // If no data got found it means that the execution can not be started via a webhook. + // Return 404 because we do not want to give any data if the execution exists or not. + const errorMessage = `The execution "${executionId}" with webhook suffix path "${path}" is not known.`; + throw new ResponseHelper.ResponseError(errorMessage, 404, 404); + } + + const workflowStartNode = workflow.getNode(lastNodeExecuted); + + if (workflowStartNode === null) { + throw new ResponseHelper.ResponseError('Could not find node to process webhook.', 404, 404); + } + + const runExecutionData = fullExecutionData.data as IRunExecutionData; + + return new Promise((resolve, reject) => { + const executionMode = 'webhook'; + WebhookHelpers.executeWebhook(workflow, webhookData, workflowData as IWorkflowDb, workflowStartNode, executionMode, undefined, runExecutionData, fullExecutionData.id, req, res, (error: Error | null, data: object) => { + if (error !== null) { + return reject(error); + } + resolve(data); + }); + }); + + } + +} diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index 4f6c540bf8..6005f7740d 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -3,7 +3,6 @@ import { get } from 'lodash'; import { ActiveExecutions, - ExternalHooks, GenericHelpers, IExecutionDb, IResponseCallbackData, @@ -29,6 +28,7 @@ import { IRunExecutionData, IWebhookData, IWebhookResponseData, + IWorkflowDataProxyAdditionalKeys, IWorkflowExecuteAdditionalData, LoggerProxy as Logger, NodeHelpers, @@ -47,7 +47,7 @@ const activeExecutions = ActiveExecutions.getInstance(); * @param {Workflow} workflow * @returns {IWebhookData[]} */ -export function getWorkflowWebhooks(workflow: Workflow, additionalData: IWorkflowExecuteAdditionalData, destinationNode?: string): IWebhookData[] { +export function getWorkflowWebhooks(workflow: Workflow, additionalData: IWorkflowExecuteAdditionalData, destinationNode?: string, ignoreRestartWehbooks = false): IWebhookData[] { // Check all the nodes in the workflow if they have webhooks const returnData: IWebhookData[] = []; @@ -65,7 +65,7 @@ export function getWorkflowWebhooks(workflow: Workflow, additionalData: IWorkflo // and no other ones continue; } - returnData.push.apply(returnData, NodeHelpers.getNodeWebhooks(workflow, node, additionalData)); + returnData.push.apply(returnData, NodeHelpers.getNodeWebhooks(workflow, node, additionalData, ignoreRestartWehbooks)); } return returnData; @@ -106,7 +106,7 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { * @param {((error: Error | null, data: IResponseCallbackData) => void)} responseCallback * @returns {(Promise)} */ - export async function executeWebhook(workflow: Workflow, webhookData: IWebhookData, workflowData: IWorkflowDb, workflowStartNode: INode, executionMode: WorkflowExecuteMode, sessionId: string | undefined, req: express.Request, res: express.Response, responseCallback: (error: Error | null, data: IResponseCallbackData) => void): Promise { +export async function executeWebhook(workflow: Workflow, webhookData: IWebhookData, workflowData: IWorkflowDb, workflowStartNode: INode, executionMode: WorkflowExecuteMode, sessionId: string | undefined, runExecutionData: IRunExecutionData | undefined, executionId: string | undefined, req: express.Request, res: express.Response, responseCallback: (error: Error | null, data: IResponseCallbackData) => void): Promise { // Get the nodeType to know which responseMode is set const nodeType = workflow.nodeTypes.getByName(workflowStartNode.type); if (nodeType === undefined) { @@ -115,9 +115,13 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { throw new ResponseHelper.ResponseError(errorMessage, 500, 500); } + const additionalKeys: IWorkflowDataProxyAdditionalKeys = { + $executionId: executionId, + }; + // Get the responseMode - const responseMode = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseMode'], executionMode, 'onReceived'); - const responseCode = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseCode'], executionMode, 200) as number; + const responseMode = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseMode'], executionMode, additionalKeys, 'onReceived'); + const responseCode = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseCode'], executionMode, additionalKeys, 200) as number; if (!['onReceived', 'lastNode'].includes(responseMode as string)) { // If the mode is not known we error. Is probably best like that instead of using @@ -129,8 +133,7 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { } // Prepare everything that is needed to run the workflow - const credentials = await WorkflowCredentials(workflowData.nodes); - const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials); + const additionalData = await WorkflowExecuteAdditionalData.getBase(); // Add the Response and Request so that this data can be accessed in the node additionalData.httpRequest = req; @@ -175,8 +178,12 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { // Save static data if it changed await WorkflowHelpers.saveStaticData(workflow); + const additionalKeys: IWorkflowDataProxyAdditionalKeys = { + $executionId: executionId, + }; + if (webhookData.webhookDescription['responseHeaders'] !== undefined) { - const responseHeaders = workflow.expression.getComplexParameterValue(workflowStartNode, webhookData.webhookDescription['responseHeaders'], executionMode, undefined) as { + const responseHeaders = workflow.expression.getComplexParameterValue(workflowStartNode, webhookData.webhookDescription['responseHeaders'], executionMode, additionalKeys, undefined) as { entries?: Array<{ name: string; value: string; @@ -257,7 +264,7 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { } ); - const runExecutionData: IRunExecutionData = { + runExecutionData = runExecutionData || { startData: { }, resultData: { @@ -268,7 +275,13 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { nodeExecutionStack, waitingExecution: {}, }, - }; + } as IRunExecutionData; + + if (executionId !== undefined) { + // Set the data the webhook node did return on the waiting node if executionId + // already exists as it means that we are restarting an existing execution. + runExecutionData.executionData!.nodeExecutionStack[0].data.main = webhookResultData.workflowData; + } if (Object.keys(runExecutionDataMerge).length !== 0) { // If data to merge got defined add it to the execution data @@ -276,7 +289,6 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { } const runData: IWorkflowExecutionDataProcess = { - credentials, executionMode, executionData: runExecutionData, sessionId, @@ -285,7 +297,7 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { // Start now to run the workflow const workflowRunner = new WorkflowRunner(); - const executionId = await workflowRunner.run(runData, true, !didSendResponse); + executionId = await workflowRunner.run(runData, true, !didSendResponse, executionId); Logger.verbose(`Started execution of workflow "${workflow.name}" from webhook with execution ID ${executionId}`, { executionId }); @@ -332,7 +344,11 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { return data; } - const responseData = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseData'], executionMode, 'firstEntryJson'); + const additionalKeys: IWorkflowDataProxyAdditionalKeys = { + $executionId: executionId, + }; + + const responseData = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseData'], executionMode, additionalKeys, 'firstEntryJson'); if (didSendResponse === false) { let data: IDataObject | IDataObject[]; @@ -347,13 +363,13 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { data = returnData.data!.main[0]![0].json; - const responsePropertyName = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responsePropertyName'], executionMode, undefined); + const responsePropertyName = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responsePropertyName'], executionMode, additionalKeys, undefined); if (responsePropertyName !== undefined) { data = get(data, responsePropertyName as string) as IDataObject; } - const responseContentType = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseContentType'], executionMode, undefined); + const responseContentType = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseContentType'], executionMode, additionalKeys, undefined); if (responseContentType !== undefined) { // Send the webhook response manually to be able to set the content-type @@ -386,7 +402,7 @@ export function getWorkflowWebhooksBasic(workflow: Workflow): IWebhookData[] { didSendResponse = true; } - const responseBinaryPropertyName = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseBinaryPropertyName'], executionMode, 'data'); + const responseBinaryPropertyName = workflow.expression.getSimpleParameterValue(workflowStartNode, webhookData.webhookDescription['responseBinaryPropertyName'], executionMode, additionalKeys, 'data'); if (responseBinaryPropertyName === undefined && didSendResponse === false) { responseCallback(new Error('No "responseBinaryPropertyName" is set.'), {}); diff --git a/packages/cli/src/WebhookServer.ts b/packages/cli/src/WebhookServer.ts index 245e3a2504..83c28ab2d0 100644 --- a/packages/cli/src/WebhookServer.ts +++ b/packages/cli/src/WebhookServer.ts @@ -26,6 +26,11 @@ import * as config from '../config'; import * as parseUrl from 'parseurl'; export function registerProductionWebhooks() { + + // ---------------------------------------- + // Regular Webhooks + // ---------------------------------------- + // HEAD webhook requests this.app.head(`/${this.endpointWebhook}/*`, async (req: express.Request, res: express.Response) => { // Cut away the "/webhook/" to get the registred part of the url diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 5c5ddfe8af..cc25661354 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -256,7 +256,7 @@ export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowEx if (execution === undefined) { // Something went badly wrong if this happens. // This check is here mostly to make typescript happy. - return undefined; + return; } const fullExecutionData: IExecutionResponse = ResponseHelper.unflattenExecutionData(execution); @@ -267,11 +267,9 @@ export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowEx return; } - if (fullExecutionData.data === undefined) { fullExecutionData.data = { - startData: { - }, + startData: {}, resultData: { runData: {}, }, @@ -351,7 +349,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { saveManualExecutions = this.workflowData.settings.saveManualExecutions as boolean; } - if (isManualMode && saveManualExecutions === false) { + if (isManualMode && saveManualExecutions === false && !fullRunData.waitTill) { // Data is always saved, so we remove from database await Db.collections.Execution!.delete(this.executionId); return; @@ -369,12 +367,14 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { if (workflowDidSucceed === true && saveDataSuccessExecution === 'none' || workflowDidSucceed === false && saveDataErrorExecution === 'none' ) { - if (!isManualMode) { - executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf); + if (!fullRunData.waitTill) { + if (!isManualMode) { + executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf); + } + // Data is always saved, so we remove from database + await Db.collections.Execution!.delete(this.executionId); + return; } - // Data is always saved, so we remove from database - await Db.collections.Execution!.delete(this.executionId); - return; } const fullExecutionData: IExecutionDb = { @@ -384,6 +384,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { startedAt: fullRunData.startedAt, stoppedAt: fullRunData.stoppedAt, workflowData: this.workflowData, + waitTill: fullRunData.waitTill, }; if (this.retryOf !== undefined) { @@ -469,6 +470,7 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { startedAt: fullRunData.startedAt, stoppedAt: fullRunData.stoppedAt, workflowData: this.workflowData, + waitTill: fullRunData.data.waitTill, }; if (this.retryOf !== undefined) { @@ -545,12 +547,7 @@ export async function getRunData(workflowData: IWorkflowBase, inputData?: INodeE }, }; - // Get the needed credentials for the current workflow as they will differ to the ones of the - // calling workflow. - const credentials = await WorkflowCredentials(workflowData!.nodes); - const runData: IWorkflowExecutionDataProcess = { - credentials, executionMode: mode, executionData: runExecutionData, // @ts-ignore @@ -618,13 +615,9 @@ export async function executeWorkflow(workflowInfo: IExecuteWorkflowInfo, additi let data; try { - // Get the needed credentials for the current workflow as they will differ to the ones of the - // calling workflow. - const credentials = await WorkflowCredentials(workflowData!.nodes); - // Create new additionalData to have different workflow loaded and to call // different webooks - const additionalDataIntegrated = await getBase(credentials); + const additionalDataIntegrated = await getBase(); additionalDataIntegrated.hooks = getWorkflowHooksIntegrated(runData.executionMode, executionId, workflowData!, { parentProcessMode: additionalData.hooks!.mode }); // Make sure we pass on the original executeWorkflow function we received // This one already contains changes to talk to parent process @@ -735,11 +728,12 @@ export function sendMessageToUI(source: string, message: any) { // tslint:disabl * @param {INodeParameters} currentNodeParameters * @returns {Promise} */ -export async function getBase(credentials: IWorkflowCredentials, currentNodeParameters?: INodeParameters, executionTimeoutTimestamp?: number): Promise { +export async function getBase(currentNodeParameters?: INodeParameters, executionTimeoutTimestamp?: number): Promise { const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl(); const timezone = config.get('generic.timezone') as string; const webhookBaseUrl = urlBaseWebhook + config.get('endpoints.webhook') as string; + const webhookWaitingBaseUrl = urlBaseWebhook + config.get('endpoints.webhookWaiting') as string; const webhookTestBaseUrl = urlBaseWebhook + config.get('endpoints.webhookTest') as string; const encryptionKey = await UserSettings.getEncryptionKey(); @@ -748,13 +742,13 @@ export async function getBase(credentials: IWorkflowCredentials, currentNodePara } return { - credentials, - credentialsHelper: new CredentialsHelper(credentials, encryptionKey), + credentialsHelper: new CredentialsHelper(encryptionKey), encryptionKey, executeWorkflow, restApiUrl: urlBaseWebhook + config.get('endpoints.rest') as string, timezone, webhookBaseUrl, + webhookWaitingBaseUrl, webhookTestBaseUrl, currentNodeParameters, executionTimeoutTimestamp, diff --git a/packages/cli/src/WorkflowHelpers.ts b/packages/cli/src/WorkflowHelpers.ts index 68718dc4cf..87e5fe3366 100644 --- a/packages/cli/src/WorkflowHelpers.ts +++ b/packages/cli/src/WorkflowHelpers.ts @@ -144,10 +144,7 @@ export async function executeErrorWorkflow(workflowId: string, workflowErrorData }, }; - const credentials = await WorkflowCredentials(workflowData.nodes); - const runData: IWorkflowExecutionDataProcess = { - credentials, executionMode, executionData: runExecutionData, workflowData, diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index 07f9b91fe2..9a8c66430f 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -123,19 +123,18 @@ export class WorkflowRunner { * @returns {Promise} * @memberof WorkflowRunner */ - async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean): Promise { + async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean, executionId?: string): Promise { const executionsProcess = config.get('executions.process') as string; const executionsMode = config.get('executions.mode') as string; - let executionId: string; if (executionsMode === 'queue' && data.executionMode !== 'manual') { // Do not run "manual" executions in bull because sending events to the // frontend would not be possible - executionId = await this.runBull(data, loadStaticData, realtime); + executionId = await this.runBull(data, loadStaticData, realtime, executionId); } else if (executionsProcess === 'main') { - executionId = await this.runMainProcess(data, loadStaticData); + executionId = await this.runMainProcess(data, loadStaticData, executionId); } else { - executionId = await this.runSubprocess(data, loadStaticData); + executionId = await this.runSubprocess(data, loadStaticData, executionId); } const externalHooks = ExternalHooks(); @@ -162,7 +161,7 @@ export class WorkflowRunner { * @returns {Promise} * @memberof WorkflowRunner */ - async runMainProcess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise { + async runMainProcess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, restartExecutionId?: string): Promise { if (loadStaticData === true && data.workflowData.id) { data.workflowData.staticData = await WorkflowHelpers.getStaticDataById(data.workflowData.id as string); } @@ -183,10 +182,13 @@ export class WorkflowRunner { } const workflow = new Workflow({ id: data.workflowData.id as string | undefined, name: data.workflowData.name, nodes: data.workflowData!.nodes, connections: data.workflowData!.connections, active: data.workflowData!.active, nodeTypes, staticData: data.workflowData!.staticData }); - const additionalData = await WorkflowExecuteAdditionalData.getBase(data.credentials, undefined, workflowTimeout <= 0 ? undefined : Date.now() + workflowTimeout * 1000); + const additionalData = await WorkflowExecuteAdditionalData.getBase(undefined, workflowTimeout <= 0 ? undefined : Date.now() + workflowTimeout * 1000); // Register the active execution - const executionId = await this.activeExecutions.add(data, undefined); + const executionId = await this.activeExecutions.add(data, undefined, restartExecutionId) as string; + additionalData.executionId = executionId; + + Logger.verbose(`Execution for workflow ${data.workflowData.name} was assigned id ${executionId}`, {executionId}); let workflowExecution: PCancelable; try { @@ -240,12 +242,12 @@ export class WorkflowRunner { return executionId; } - async runBull(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean): Promise { + async runBull(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean, restartExecutionId?: string): Promise { // TODO: If "loadStaticData" is set to true it has to load data new on worker // Register the active execution - const executionId = await this.activeExecutions.add(data, undefined); + const executionId = await this.activeExecutions.add(data, undefined, restartExecutionId); const jobData: IBullJobData = { executionId, @@ -412,7 +414,7 @@ export class WorkflowRunner { * @returns {Promise} * @memberof WorkflowRunner */ - async runSubprocess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise { + async runSubprocess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, restartExecutionId?: string): Promise { let startedAt = new Date(); const subprocess = fork(pathJoin(__dirname, 'WorkflowRunnerProcess.js')); @@ -421,45 +423,16 @@ export class WorkflowRunner { } // Register the active execution - const executionId = await this.activeExecutions.add(data, subprocess); + const executionId = await this.activeExecutions.add(data, subprocess, restartExecutionId); - // Check if workflow contains a "executeWorkflow" Node as in this - // case we can not know which nodeTypes and credentialTypes will - // be needed and so have to load all of them in the workflowRunnerProcess - let loadAllNodeTypes = false; - for (const node of data.workflowData.nodes) { - if (node.type === 'n8n-nodes-base.executeWorkflow') { - loadAllNodeTypes = true; - break; - } - } - - let nodeTypeData: ITransferNodeTypes; - let credentialTypeData: ICredentialsTypeData; - let credentialsOverwrites = this.credentialsOverwrites; - - if (loadAllNodeTypes === true) { - // Supply all nodeTypes and credentialTypes - nodeTypeData = WorkflowHelpers.getAllNodeTypeData(); - const credentialTypes = CredentialTypes(); - credentialTypeData = credentialTypes.credentialTypes; - } else { - // Supply only nodeTypes, credentialTypes and overwrites that the workflow needs - nodeTypeData = WorkflowHelpers.getNodeTypeData(data.workflowData.nodes); - credentialTypeData = WorkflowHelpers.getCredentialsData(data.credentials); - - credentialsOverwrites = {}; - for (const credentialName of Object.keys(credentialTypeData)) { - if (this.credentialsOverwrites[credentialName] !== undefined) { - credentialsOverwrites[credentialName] = this.credentialsOverwrites[credentialName]; - } - } - } + // Supply all nodeTypes and credentialTypes + const nodeTypeData = WorkflowHelpers.getAllNodeTypeData() as ITransferNodeTypes; + const credentialTypes = CredentialTypes(); (data as unknown as IWorkflowExecutionDataProcessWithExecution).executionId = executionId; (data as unknown as IWorkflowExecutionDataProcessWithExecution).nodeTypeData = nodeTypeData; - (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsOverwrite = credentialsOverwrites; - (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsTypeData = credentialTypeData; // TODO: Still needs correct value + (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsOverwrite = this.credentialsOverwrites; + (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsTypeData = credentialTypes.credentialTypes; const workflowHooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId); diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index 321389ae2f..6b3262322b 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -111,9 +111,22 @@ export class WorkflowRunnerProcess { const externalHooks = ExternalHooks(); await externalHooks.init(); - // This code has been split into 3 ifs just to make it easier to understand + // Credentials should now be loaded from database. + // We check if any node uses credentials. If it does, then + // init database. + let shouldInitializaDb = false; + inputData.workflowData.nodes.map(node => { + if (Object.keys(node.credentials === undefined ? {} : node.credentials).length > 0) { + shouldInitializaDb = true; + } + }); + + // This code has been split into 4 ifs just to make it easier to understand // Can be made smaller but in the end it will make it impossible to read. - if (inputData.workflowData.settings !== undefined && inputData.workflowData.settings.saveExecutionProgress === true) { + if (shouldInitializaDb) { + // initialize db as we need to load credentials + await Db.init(); + } else if (inputData.workflowData.settings !== undefined && inputData.workflowData.settings.saveExecutionProgress === true) { // Workflow settings specifying it should save await Db.init(); } else if (inputData.workflowData.settings !== undefined && inputData.workflowData.settings.saveExecutionProgress !== false && config.get('executions.saveExecutionProgress') as boolean) { @@ -135,8 +148,9 @@ export class WorkflowRunnerProcess { } this.workflow = new Workflow({ id: this.data.workflowData.id as string | undefined, name: this.data.workflowData.name, nodes: this.data.workflowData!.nodes, connections: this.data.workflowData!.connections, active: this.data.workflowData!.active, nodeTypes, staticData: this.data.workflowData!.staticData, settings: this.data.workflowData!.settings }); - const additionalData = await WorkflowExecuteAdditionalData.getBase(this.data.credentials, undefined, workflowTimeout <= 0 ? undefined : Date.now() + workflowTimeout * 1000); + const additionalData = await WorkflowExecuteAdditionalData.getBase(undefined, workflowTimeout <= 0 ? undefined : Date.now() + workflowTimeout * 1000); additionalData.hooks = this.getProcessForwardHooks(); + additionalData.executionId = inputData.executionId; additionalData.sendMessageToUI = async (source: string, message: any) => { // tslint:disable-line:no-any if (workflowRunner.data!.executionMode !== 'manual') { diff --git a/packages/cli/src/databases/entities/ExecutionEntity.ts b/packages/cli/src/databases/entities/ExecutionEntity.ts index b788f5e153..ba6b60807f 100644 --- a/packages/cli/src/databases/entities/ExecutionEntity.ts +++ b/packages/cli/src/databases/entities/ExecutionEntity.ts @@ -53,4 +53,8 @@ export class ExecutionEntity implements IExecutionFlattedDb { @Index() @Column({ nullable: true }) workflowId: string; + + @Index() + @Column({ type: resolveDataType('datetime') as ColumnOptions['type'], nullable: true }) + waitTill: Date; } diff --git a/packages/cli/src/databases/mysqldb/migrations/1626183952959-AddWaitColumn.ts b/packages/cli/src/databases/mysqldb/migrations/1626183952959-AddWaitColumn.ts new file mode 100644 index 0000000000..ee2aa560e1 --- /dev/null +++ b/packages/cli/src/databases/mysqldb/migrations/1626183952959-AddWaitColumn.ts @@ -0,0 +1,22 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; +import * as config from '../../../../config'; + +export class AddWaitColumnId1626183952959 implements MigrationInterface { + name = 'AddWaitColumnId1626183952959'; + + async up(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + await queryRunner.query('ALTER TABLE `' + tablePrefix + 'execution_entity` ADD `waitTill` DATETIME NULL'); + await queryRunner.query('CREATE INDEX `IDX_' + tablePrefix + 'ca4a71b47f28ac6ea88293a8e2` ON `' + tablePrefix + 'execution_entity` (`waitTill`)'); + } + + async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + await queryRunner.query( + 'DROP INDEX `IDX_' + tablePrefix + 'ca4a71b47f28ac6ea88293a8e2` ON `' + tablePrefix + 'execution_entity`' + ); + await queryRunner.query('ALTER TABLE `' + tablePrefix + 'execution_entity` DROP COLUMN `waitTill`'); + } +} diff --git a/packages/cli/src/databases/mysqldb/migrations/index.ts b/packages/cli/src/databases/mysqldb/migrations/index.ts index 1054d68cfc..b48bc58aff 100644 --- a/packages/cli/src/databases/mysqldb/migrations/index.ts +++ b/packages/cli/src/databases/mysqldb/migrations/index.ts @@ -8,6 +8,7 @@ import { ChangeCredentialDataSize1620729500000 } from './1620729500000-ChangeCre import { CreateTagEntity1617268711084 } from './1617268711084-CreateTagEntity'; import { UniqueWorkflowNames1620826335440 } from './1620826335440-UniqueWorkflowNames'; import { CertifyCorrectCollation1623936588000 } from './1623936588000-CertifyCorrectCollation'; +import { AddWaitColumnId1626183952959 } from './1626183952959-AddWaitColumn'; export const mysqlMigrations = [ InitialMigration1588157391238, @@ -20,4 +21,5 @@ export const mysqlMigrations = [ CreateTagEntity1617268711084, UniqueWorkflowNames1620826335440, CertifyCorrectCollation1623936588000, + AddWaitColumnId1626183952959, ]; diff --git a/packages/cli/src/databases/postgresdb/migrations/1626176912946-AddwaitTill.ts b/packages/cli/src/databases/postgresdb/migrations/1626176912946-AddwaitTill.ts new file mode 100644 index 0000000000..3bef043a83 --- /dev/null +++ b/packages/cli/src/databases/postgresdb/migrations/1626176912946-AddwaitTill.ts @@ -0,0 +1,31 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; +import * as config from '../../../../config'; + +export class AddwaitTill1626176912946 implements MigrationInterface { + name = 'AddwaitTill1626176912946'; + + async up(queryRunner: QueryRunner): Promise { + let tablePrefix = config.get('database.tablePrefix'); + const tablePrefixPure = tablePrefix; + const schema = config.get('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } + + await queryRunner.query(`ALTER TABLE ${tablePrefix}execution_entity ADD "waitTill" TIMESTAMP`); + await queryRunner.query(`CREATE INDEX IF NOT EXISTS IDX_${tablePrefixPure}ca4a71b47f28ac6ea88293a8e2 ON ${tablePrefix}execution_entity ("waitTill")`); + } + + async down(queryRunner: QueryRunner): Promise { + let tablePrefix = config.get('database.tablePrefix'); + const tablePrefixPure = tablePrefix; + const schema = config.get('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } + + await queryRunner.query(`DROP INDEX IDX_${tablePrefixPure}ca4a71b47f28ac6ea88293a8e2`); + await queryRunner.query(`ALTER TABLE ${tablePrefix}webhook_entity DROP COLUMN "waitTill"`); + } + +} diff --git a/packages/cli/src/databases/postgresdb/migrations/index.ts b/packages/cli/src/databases/postgresdb/migrations/index.ts index 0f6cc669c9..83983dd039 100644 --- a/packages/cli/src/databases/postgresdb/migrations/index.ts +++ b/packages/cli/src/databases/postgresdb/migrations/index.ts @@ -5,6 +5,7 @@ import { AddWebhookId1611144599516 } from './1611144599516-AddWebhookId'; import { MakeStoppedAtNullable1607431743768 } from './1607431743768-MakeStoppedAtNullable'; import { CreateTagEntity1617270242566 } from './1617270242566-CreateTagEntity'; import { UniqueWorkflowNames1620824779533 } from './1620824779533-UniqueWorkflowNames'; +import { AddwaitTill1626176912946 } from './1626176912946-AddwaitTill'; export const postgresMigrations = [ InitialMigration1587669153312, @@ -14,4 +15,5 @@ export const postgresMigrations = [ MakeStoppedAtNullable1607431743768, CreateTagEntity1617270242566, UniqueWorkflowNames1620824779533, + AddwaitTill1626176912946, ]; diff --git a/packages/cli/src/databases/sqlite/migrations/1621707690587-AddWaitColumn.ts b/packages/cli/src/databases/sqlite/migrations/1621707690587-AddWaitColumn.ts new file mode 100644 index 0000000000..d4d10bae02 --- /dev/null +++ b/packages/cli/src/databases/sqlite/migrations/1621707690587-AddWaitColumn.ts @@ -0,0 +1,32 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import * as config from '../../../../config'; + +export class AddWaitColumn1621707690587 implements MigrationInterface { + name = 'AddWaitColumn1621707690587'; + + async up(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + await queryRunner.query(`DROP TABLE IF EXISTS "${tablePrefix}temporary_execution_entity"`); + await queryRunner.query(`CREATE TABLE "${tablePrefix}temporary_execution_entity" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "data" text NOT NULL, "finished" boolean NOT NULL, "mode" varchar NOT NULL, "retryOf" varchar, "retrySuccessId" varchar, "startedAt" datetime NOT NULL, "stoppedAt" datetime, "workflowData" text NOT NULL, "workflowId" varchar, "waitTill" DATETIME)`, undefined); + await queryRunner.query(`INSERT INTO "${tablePrefix}temporary_execution_entity"("id", "data", "finished", "mode", "retryOf", "retrySuccessId", "startedAt", "stoppedAt", "workflowData", "workflowId") SELECT "id", "data", "finished", "mode", "retryOf", "retrySuccessId", "startedAt", "stoppedAt", "workflowData", "workflowId" FROM "${tablePrefix}execution_entity"`); + await queryRunner.query(`DROP TABLE "${tablePrefix}execution_entity"`); + await queryRunner.query(`ALTER TABLE "${tablePrefix}temporary_execution_entity" RENAME TO "${tablePrefix}execution_entity"`); + await queryRunner.query(`CREATE INDEX "IDX_${tablePrefix}cefb067df2402f6aed0638a6c1" ON "${tablePrefix}execution_entity" ("stoppedAt")`); + await queryRunner.query(`CREATE INDEX "IDX_${tablePrefix}ca4a71b47f28ac6ea88293a8e2" ON "${tablePrefix}execution_entity" ("waitTill")`); + await queryRunner.query(`VACUUM;`); + } + + async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + await queryRunner.query(`CREATE TABLE IF NOT EXISTS "${tablePrefix}temporary_execution_entity" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "data" text NOT NULL, "finished" boolean NOT NULL, "mode" varchar NOT NULL, "retryOf" varchar, "retrySuccessId" varchar, "startedAt" datetime NOT NULL, "stoppedAt" datetime, "workflowData" text NOT NULL, "workflowId" varchar)`, undefined); + await queryRunner.query(`INSERT INTO "${tablePrefix}temporary_execution_entity"("id", "data", "finished", "mode", "retryOf", "retrySuccessId", "startedAt", "stoppedAt", "workflowData", "workflowId") SELECT "id", "data", "finished", "mode", "retryOf", "retrySuccessId", "startedAt", "stoppedAt", "workflowData", "workflowId" FROM "${tablePrefix}execution_entity"`); + await queryRunner.query(`DROP TABLE "${tablePrefix}execution_entity"`); + await queryRunner.query(`ALTER TABLE "${tablePrefix}temporary_execution_entity" RENAME TO "${tablePrefix}execution_entity"`); + await queryRunner.query(`CREATE INDEX "IDX_${tablePrefix}cefb067df2402f6aed0638a6c1" ON "${tablePrefix}execution_entity" ("stoppedAt")`); + await queryRunner.query(`VACUUM;`); + + } + +} diff --git a/packages/cli/src/databases/sqlite/migrations/index.ts b/packages/cli/src/databases/sqlite/migrations/index.ts index 14b5184954..64038d9e30 100644 --- a/packages/cli/src/databases/sqlite/migrations/index.ts +++ b/packages/cli/src/databases/sqlite/migrations/index.ts @@ -5,6 +5,7 @@ import { AddWebhookId1611071044839 } from './1611071044839-AddWebhookId'; import { MakeStoppedAtNullable1607431743769 } from './1607431743769-MakeStoppedAtNullable'; import { CreateTagEntity1617213344594 } from './1617213344594-CreateTagEntity'; import { UniqueWorkflowNames1620821879465 } from './1620821879465-UniqueWorkflowNames'; +import { AddWaitColumn1621707690587 } from './1621707690587-AddWaitColumn'; export const sqliteMigrations = [ InitialMigration1588102412422, @@ -14,4 +15,5 @@ export const sqliteMigrations = [ MakeStoppedAtNullable1607431743769, CreateTagEntity1617213344594, UniqueWorkflowNames1620821879465, + AddWaitColumn1621707690587, ]; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index cc7e942fe5..296348cf65 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -5,6 +5,8 @@ export * from './ExternalHooks'; export * from './Interfaces'; export * from './LoadNodesAndCredentials'; export * from './NodeTypes'; +export * from './WaitTracker'; +export * from './WaitingWebhooks'; export * from './WorkflowCredentials'; export * from './WorkflowRunner'; diff --git a/packages/core/package.json b/packages/core/package.json index bca7d0dffc..ee2c8da502 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "n8n-core", - "version": "0.79.0", + "version": "0.80.0", "description": "Core functionality of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", @@ -50,7 +50,7 @@ "file-type": "^14.6.2", "lodash.get": "^4.4.2", "mime-types": "^2.1.27", - "n8n-workflow": "~0.64.0", + "n8n-workflow": "~0.65.0", "oauth-1.0a": "^2.2.6", "p-cancelable": "^2.0.0", "request": "^2.88.2", diff --git a/packages/core/src/Constants.ts b/packages/core/src/Constants.ts index a11b6d9402..4e72f53f9c 100644 --- a/packages/core/src/Constants.ts +++ b/packages/core/src/Constants.ts @@ -5,4 +5,6 @@ export const EXTENSIONS_SUBDIRECTORY = 'custom'; export const USER_FOLDER_ENV_OVERWRITE = 'N8N_USER_FOLDER'; export const USER_SETTINGS_FILE_NAME = 'config'; export const USER_SETTINGS_SUBFOLDER = '.n8n'; +export const PLACEHOLDER_EMPTY_EXECUTION_ID = '__UNKOWN__'; export const TUNNEL_SUBDOMAIN_ENV = 'N8N_TUNNEL_SUBDOMAIN'; +export const WAIT_TIME_UNLIMITED = '3000-01-01T00:00:00.000Z'; diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index 763afdf4b8..52bd80f700 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -34,9 +34,10 @@ export interface IProcessMessage { export interface IExecuteFunctions extends IExecuteFunctionsBase { helpers: { prepareBinaryData(binaryData: Buffer, filePath?: string, mimeType?: string): Promise; - request: requestPromise.RequestPromiseAPI, - requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise, // tslint:disable-line:no-any - requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | requestPromise.RequestPromiseOptions): Promise, // tslint:disable-line:no-any + getBinaryDataBuffer(itemIndex: number, propertyName: string): Promise; + request: requestPromise.RequestPromiseAPI; + requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise; // tslint:disable-line:no-any + requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | requestPromise.RequestPromiseOptions): Promise; // tslint:disable-line:no-any returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExecutionData[]; }; } diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 218eb097d4..582f5ac1ca 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -4,6 +4,7 @@ import { ILoadOptionsFunctions, IResponseError, IWorkflowSettings, + PLACEHOLDER_EMPTY_EXECUTION_ID, } from './'; import { @@ -28,6 +29,7 @@ import { IWebhookData, IWebhookDescription, IWebhookFunctions, + IWorkflowDataProxyAdditionalKeys, IWorkflowDataProxyData, IWorkflowExecuteAdditionalData, IWorkflowMetadata, @@ -59,6 +61,21 @@ const requestPromiseWithDefaults = requestPromise.defaults({ timeout: 300000, // 5 minutes }); +/** + * Returns binary data buffer for given item index and property name. + * + * @export + * @param {ITaskDataConnections} inputData + * @param {number} itemIndex + * @param {string} propertyName + * @param {number} inputIndex + * @returns {Promise} + */ +export async function getBinaryDataBuffer(inputData: ITaskDataConnections, itemIndex: number, propertyName: string, inputIndex: number): Promise { + const binaryData = inputData['main']![inputIndex]![itemIndex]!.binary![propertyName]!; + return Buffer.from(binaryData.data, BINARY_ENCODING); +} + /** * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as * base64 and adds metadata. @@ -140,8 +157,8 @@ export async function prepareBinaryData(binaryData: Buffer, filePath?: string, m * * @returns */ -export function requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, node: INode, additionalData: IWorkflowExecuteAdditionalData, oAuth2Options?: IOAuth2Options) { - const credentials = this.getCredentials(credentialsType) as ICredentialDataDecryptedObject; +export async function requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, node: INode, additionalData: IWorkflowExecuteAdditionalData, oAuth2Options?: IOAuth2Options) { + const credentials = await this.getCredentials(credentialsType) as ICredentialDataDecryptedObject; if (credentials === undefined) { throw new Error('No credentials got returned!'); @@ -229,8 +246,8 @@ export function requestOAuth2(this: IAllExecuteFunctions, credentialsType: strin * @param {(OptionsWithUrl | requestPromise.RequestPromiseOptions)} requestOptionså * @returns */ -export function requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | OptionsWithUri | requestPromise.RequestPromiseOptions) { - const credentials = this.getCredentials(credentialsType) as ICredentialDataDecryptedObject; +export async function requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | OptionsWithUri | requestPromise.RequestPromiseOptions) { + const credentials = await this.getCredentials(credentialsType) as ICredentialDataDecryptedObject; if (credentials === undefined) { throw new Error('No credentials got returned!'); @@ -307,6 +324,23 @@ export function returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExe +/** + * Returns the additional keys for Expressions and Function-Nodes + * + * @export + * @param {IWorkflowExecuteAdditionalData} additionalData + * @returns {(IWorkflowDataProxyAdditionalKeys)} + */ +export function getAdditionalKeys(additionalData: IWorkflowExecuteAdditionalData): IWorkflowDataProxyAdditionalKeys { + const executionId = additionalData.executionId || PLACEHOLDER_EMPTY_EXECUTION_ID; + return { + $executionId: executionId, + $resumeWebhookUrl: `${additionalData.webhookWaitingBaseUrl}/${executionId}`, + }; +} + + + /** * Returns the requested decrypted credentials if the node has access to them. * @@ -317,7 +351,7 @@ export function returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExe * @param {IWorkflowExecuteAdditionalData} additionalData * @returns {(ICredentialDataDecryptedObject | undefined)} */ -export function getCredentials(workflow: Workflow, node: INode, type: string, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, runExecutionData?: IRunExecutionData | null, runIndex?: number, connectionInputData?: INodeExecutionData[], itemIndex?: number): ICredentialDataDecryptedObject | undefined { +export async function getCredentials(workflow: Workflow, node: INode, type: string, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, runExecutionData?: IRunExecutionData | null, runIndex?: number, connectionInputData?: INodeExecutionData[], itemIndex?: number): Promise { // Get the NodeType as it has the information if the credentials are required const nodeType = workflow.nodeTypes.getByName(node.type); @@ -371,7 +405,7 @@ export function getCredentials(workflow: Workflow, node: INode, type: string, ad const name = node.credentials[type]; - const decryptedDataObject = additionalData.credentialsHelper.getDecrypted(name, type, mode, false, expressionResolveValues); + const decryptedDataObject = await additionalData.credentialsHelper.getDecrypted(name, type, mode, false, expressionResolveValues); return decryptedDataObject; } @@ -405,7 +439,7 @@ export function getNode(node: INode): INode { * @param {*} [fallbackValue] * @returns {(NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object)} */ -export function getNodeParameter(workflow: Workflow, runExecutionData: IRunExecutionData | null, runIndex: number, connectionInputData: INodeExecutionData[], node: INode, parameterName: string, itemIndex: number, mode: WorkflowExecuteMode, fallbackValue?: any): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object { //tslint:disable-line:no-any +export function getNodeParameter(workflow: Workflow, runExecutionData: IRunExecutionData | null, runIndex: number, connectionInputData: INodeExecutionData[], node: INode, parameterName: string, itemIndex: number, mode: WorkflowExecuteMode, additionalKeys: IWorkflowDataProxyAdditionalKeys, fallbackValue?: any): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object { //tslint:disable-line:no-any const nodeType = workflow.nodeTypes.getByName(node.type); if (nodeType === undefined) { throw new Error(`Node type "${node.type}" is not known so can not return paramter value!`); @@ -419,7 +453,7 @@ export function getNodeParameter(workflow: Workflow, runExecutionData: IRunExecu let returnData; try { - returnData = workflow.expression.getParameterValue(value, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, mode); + returnData = workflow.expression.getParameterValue(value, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, mode, additionalKeys); } catch (e) { e.message += ` [Error in parameter: "${parameterName}"]`; throw e; @@ -454,7 +488,7 @@ export function continueOnFail(node: INode): boolean { * @param {boolean} [isTest] * @returns {(string | undefined)} */ -export function getNodeWebhookUrl(name: string, workflow: Workflow, node: INode, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, isTest?: boolean): string | undefined { +export function getNodeWebhookUrl(name: string, workflow: Workflow, node: INode, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, additionalKeys: IWorkflowDataProxyAdditionalKeys, isTest?: boolean): string | undefined { let baseUrl = additionalData.webhookBaseUrl; if (isTest === true) { baseUrl = additionalData.webhookTestBaseUrl; @@ -465,12 +499,12 @@ export function getNodeWebhookUrl(name: string, workflow: Workflow, node: INode, return undefined; } - const path = workflow.expression.getSimpleParameterValue(node, webhookDescription['path'], mode); + const path = workflow.expression.getSimpleParameterValue(node, webhookDescription['path'], mode, additionalKeys); if (path === undefined) { return undefined; } - const isFullPath: boolean = workflow.expression.getSimpleParameterValue(node, webhookDescription['isFullPath'], mode, false) as boolean; + const isFullPath: boolean = workflow.expression.getSimpleParameterValue(node, webhookDescription['isFullPath'], mode, additionalKeys, false) as boolean; return NodeHelpers.getNodeWebhookUrl(baseUrl, workflow.id!, node, path.toString(), isFullPath); } @@ -555,8 +589,8 @@ export function getExecutePollFunctions(workflow: Workflow, node: INode, additio __emit: (data: INodeExecutionData[][]): void => { throw new Error('Overwrite NodeExecuteFunctions.getExecutePullFunctions.__emit function!'); }, - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, mode); }, getMode: (): WorkflowExecuteMode => { return mode; @@ -573,7 +607,7 @@ export function getExecutePollFunctions(workflow: Workflow, node: INode, additio const runIndex = 0; const connectionInputData: INodeExecutionData[] = []; - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getRestApiUrl: (): string => { return additionalData.restApiUrl; @@ -621,8 +655,8 @@ export function getExecuteTriggerFunctions(workflow: Workflow, node: INode, addi emit: (data: INodeExecutionData[][]): void => { throw new Error('Overwrite NodeExecuteFunctions.getExecuteTriggerFunctions.emit function!'); }, - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, mode); }, getNode: () => { return getNode(node); @@ -639,7 +673,7 @@ export function getExecuteTriggerFunctions(workflow: Workflow, node: INode, addi const runIndex = 0; const connectionInputData: INodeExecutionData[] = []; - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getRestApiUrl: (): string => { return additionalData.restApiUrl; @@ -691,7 +725,7 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx return continueOnFail(node); }, evaluateExpression: (expression: string, itemIndex: number) => { - return workflow.expression.resolveSimpleParameterValue('=' + expression, {}, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, mode); + return workflow.expression.resolveSimpleParameterValue('=' + expression, {}, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, mode, getAdditionalKeys(additionalData)); }, async executeWorkflow(workflowInfo: IExecuteWorkflowInfo, inputData?: INodeExecutionData[]): Promise { // tslint:disable-line:no-any return additionalData.executeWorkflow(workflowInfo, additionalData, inputData); @@ -699,8 +733,11 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx getContext(type: string): IContextObject { return NodeHelpers.getContext(runExecutionData, type, node); }, - getCredentials(type: string, itemIndex?: number): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode, runExecutionData, runIndex, connectionInputData, itemIndex); + async getCredentials(type: string, itemIndex?: number): Promise { + return await getCredentials(workflow, node, type, additionalData, mode, runExecutionData, runIndex, connectionInputData, itemIndex); + }, + getExecutionId: (): string => { + return additionalData.executionId!; }, getInputData: (inputIndex = 0, inputName = 'main') => { @@ -714,17 +751,15 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx throw new Error(`Could not get input index "${inputIndex}" of input "${inputName}"!`); } - if (inputData[inputName][inputIndex] === null) { // return []; throw new Error(`Value "${inputIndex}" of input "${inputName}" did not get set!`); } - // TODO: Maybe do clone of data only here so it only clones the data that is really needed return inputData[inputName][inputIndex] as INodeExecutionData[]; }, getNodeParameter: (parameterName: string, itemIndex: number, fallbackValue?: any): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object => { //tslint:disable-line:no-any - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getMode: (): WorkflowExecuteMode => { return mode; @@ -742,14 +777,17 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx return getWorkflowMetadata(workflow); }, getWorkflowDataProxy: (itemIndex: number): IWorkflowDataProxyData => { - const dataProxy = new WorkflowDataProxy(workflow, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, {}, mode); + const dataProxy = new WorkflowDataProxy(workflow, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, {}, mode, getAdditionalKeys(additionalData)); return dataProxy.getDataProxy(); }, getWorkflowStaticData(type: string): IDataObject { return workflow.getStaticData(type, node); }, prepareOutputData: NodeHelpers.prepareOutputData, - sendMessageToUI(message: any): void { // tslint:disable-line:no-any + async putExecutionToWait(waitTill: Date): Promise { + runExecutionData.waitTill = waitTill; + }, + sendMessageToUI(message : any): void { // tslint:disable-line:no-any if (mode !== 'manual') { return; } @@ -763,6 +801,9 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx }, helpers: { prepareBinaryData, + getBinaryDataBuffer(itemIndex: number, propertyName: string, inputIndex = 0): Promise { + return getBinaryDataBuffer.call(this, inputData, itemIndex, propertyName, inputIndex); + }, request: requestPromiseWithDefaults, requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise { // tslint:disable-line:no-any return requestOAuth2.call(this, credentialsType, requestOptions, node, additionalData, oAuth2Options); @@ -801,13 +842,13 @@ export function getExecuteSingleFunctions(workflow: Workflow, runExecutionData: }, evaluateExpression: (expression: string, evaluateItemIndex: number | undefined) => { evaluateItemIndex = evaluateItemIndex === undefined ? itemIndex : evaluateItemIndex; - return workflow.expression.resolveSimpleParameterValue('=' + expression, {}, runExecutionData, runIndex, evaluateItemIndex, node.name, connectionInputData, mode); + return workflow.expression.resolveSimpleParameterValue('=' + expression, {}, runExecutionData, runIndex, evaluateItemIndex, node.name, connectionInputData, mode, getAdditionalKeys(additionalData)); }, getContext(type: string): IContextObject { return NodeHelpers.getContext(runExecutionData, type, node); }, - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode, runExecutionData, runIndex, connectionInputData, itemIndex); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, mode, runExecutionData, runIndex, connectionInputData, itemIndex); }, getInputData: (inputIndex = 0, inputName = 'main') => { if (!inputData.hasOwnProperty(inputName)) { @@ -847,13 +888,13 @@ export function getExecuteSingleFunctions(workflow: Workflow, runExecutionData: return getTimezone(workflow, additionalData); }, getNodeParameter: (parameterName: string, fallbackValue?: any): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object => { //tslint:disable-line:no-any - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getWorkflow: () => { return getWorkflowMetadata(workflow); }, getWorkflowDataProxy: (): IWorkflowDataProxyData => { - const dataProxy = new WorkflowDataProxy(workflow, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, {}, mode); + const dataProxy = new WorkflowDataProxy(workflow, runExecutionData, runIndex, itemIndex, node.name, connectionInputData, {}, mode, getAdditionalKeys(additionalData)); return dataProxy.getDataProxy(); }, getWorkflowStaticData(type: string): IDataObject { @@ -886,8 +927,8 @@ export function getExecuteSingleFunctions(workflow: Workflow, runExecutionData: export function getLoadOptionsFunctions(workflow: Workflow, node: INode, path: string, additionalData: IWorkflowExecuteAdditionalData): ILoadOptionsFunctions { return ((workflow: Workflow, node: INode, path: string) => { const that = { - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, 'internal'); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, 'internal'); }, getCurrentNodeParameter: (parameterPath: string): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object | undefined => { const nodeParameters = additionalData.currentNodeParameters; @@ -910,7 +951,7 @@ export function getLoadOptionsFunctions(workflow: Workflow, node: INode, path: s const runIndex = 0; const connectionInputData: INodeExecutionData[] = []; - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, 'internal' as WorkflowExecuteMode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, 'internal' as WorkflowExecuteMode, getAdditionalKeys(additionalData), fallbackValue); }, getTimezone: (): string => { return getTimezone(workflow, additionalData); @@ -947,8 +988,8 @@ export function getLoadOptionsFunctions(workflow: Workflow, node: INode, path: s export function getExecuteHookFunctions(workflow: Workflow, node: INode, additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, activation: WorkflowActivateMode, isTest?: boolean, webhookData?: IWebhookData): IHookFunctions { return ((workflow: Workflow, node: INode) => { const that = { - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, mode); }, getMode: (): WorkflowExecuteMode => { return mode; @@ -965,10 +1006,10 @@ export function getExecuteHookFunctions(workflow: Workflow, node: INode, additio const runIndex = 0; const connectionInputData: INodeExecutionData[] = []; - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getNodeWebhookUrl: (name: string): string | undefined => { - return getNodeWebhookUrl(name, workflow, node, additionalData, mode, isTest); + return getNodeWebhookUrl(name, workflow, node, additionalData, mode, getAdditionalKeys(additionalData), isTest); }, getTimezone: (): string => { return getTimezone(workflow, additionalData); @@ -1024,8 +1065,8 @@ export function getExecuteWebhookFunctions(workflow: Workflow, node: INode, addi } return additionalData.httpRequest.body; }, - getCredentials(type: string): ICredentialDataDecryptedObject | undefined { - return getCredentials(workflow, node, type, additionalData, mode); + async getCredentials(type: string): Promise { + return await getCredentials(workflow, node, type, additionalData, mode); }, getHeaderData(): object { if (additionalData.httpRequest === undefined) { @@ -1045,7 +1086,7 @@ export function getExecuteWebhookFunctions(workflow: Workflow, node: INode, addi const runIndex = 0; const connectionInputData: INodeExecutionData[] = []; - return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, fallbackValue); + return getNodeParameter(workflow, runExecutionData, runIndex, connectionInputData, node, parameterName, itemIndex, mode, getAdditionalKeys(additionalData), fallbackValue); }, getParamsData(): object { if (additionalData.httpRequest === undefined) { @@ -1072,7 +1113,7 @@ export function getExecuteWebhookFunctions(workflow: Workflow, node: INode, addi return additionalData.httpResponse; }, getNodeWebhookUrl: (name: string): string | undefined => { - return getNodeWebhookUrl(name, workflow, node, additionalData, mode); + return getNodeWebhookUrl(name, workflow, node, additionalData, mode, getAdditionalKeys(additionalData)); }, getTimezone: (): string => { return getTimezone(workflow, additionalData); diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index b893b85cb0..a3e78b022b 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -31,7 +31,6 @@ export class WorkflowExecute { private additionalData: IWorkflowExecuteAdditionalData; private mode: WorkflowExecuteMode; - constructor(additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, runExecutionData?: IRunExecutionData) { this.additionalData = additionalData; this.mode = mode; @@ -512,6 +511,13 @@ export class WorkflowExecute { this.runExecutionData.startData = {}; } + if (this.runExecutionData.waitTill) { + const lastNodeExecuted = this.runExecutionData.resultData.lastNodeExecuted as string; + this.runExecutionData.executionData!.nodeExecutionStack[0].node.disabled = true; + this.runExecutionData.waitTill = undefined; + this.runExecutionData.resultData.runData[lastNodeExecuted].pop(); + } + let currentExecutionTry = ''; let lastExecutionTry = ''; @@ -693,7 +699,7 @@ export class WorkflowExecute { } } - if (nodeSuccessData === null) { + if (nodeSuccessData === null && !this.runExecutionData.waitTill!!) { // If null gets returned it means that the node did succeed // but did not have any data. So the branch should end // (meaning the nodes afterwards should not be processed) @@ -767,6 +773,15 @@ export class WorkflowExecute { continue; } + if (this.runExecutionData.waitTill!!) { + await this.executeHook('nodeExecuteAfter', [executionNode.name, taskData, this.runExecutionData]); + + // Add the node back to the stack that the workflow can start to execute again from that node + this.runExecutionData.executionData!.nodeExecutionStack.unshift(executionData); + + break; + } + // Add the nodes to which the current node has an output connection to that they can // be executed next if (workflow.connectionsBySourceNode.hasOwnProperty(executionNode.name)) { @@ -849,6 +864,9 @@ export class WorkflowExecute { message: executionError.message, stack: executionError.stack, } as ExecutionError; + } else if (this.runExecutionData.waitTill!!) { + Logger.verbose(`Workflow execution will wait until ${this.runExecutionData.waitTill}`, { workflowId: workflow.id }); + fullRunData.waitTill = this.runExecutionData.waitTill; } else { Logger.verbose(`Workflow execution finished successfully`, { workflowId: workflow.id }); fullRunData.finished = true; diff --git a/packages/core/test/Helpers.ts b/packages/core/test/Helpers.ts index bba3a3452d..ce59151abc 100644 --- a/packages/core/test/Helpers.ts +++ b/packages/core/test/Helpers.ts @@ -26,12 +26,14 @@ import { export class CredentialsHelper extends ICredentialsHelper { - getDecrypted(name: string, type: string): ICredentialDataDecryptedObject { - return {}; + getDecrypted(name: string, type: string): Promise { + return new Promise(res => res({})); } - getCredentials(name: string, type: string): Credentials { - return new Credentials('', '', [], ''); + getCredentials(name: string, type: string): Promise { + return new Promise(res => { + res(new Credentials('', '', [], '')); + }); } async updateCredentials(name: string, type: string, data: ICredentialDataDecryptedObject): Promise {} @@ -748,8 +750,7 @@ export function WorkflowExecuteAdditionalData(waitPromise: IDeferredPromise => {}, // tslint:disable-line:no-any sendMessageToUI: (message: string) => {}, @@ -757,6 +758,7 @@ export function WorkflowExecuteAdditionalData(waitPromise: IDeferredPromise { const workflowExecute = new WorkflowExecute(additionalData, executionMode); - const executionData = await workflowExecute.run(workflowInstance, undefined); + const executionData = await workflowExecute.run(workflowInstance); const result = await waitPromise.promise(); diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index da30296d1f..2d0ac3a7c5 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -1,6 +1,6 @@ { "name": "n8n-editor-ui", - "version": "0.102.0", + "version": "0.103.0", "description": "Workflow Editor UI for n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", @@ -72,7 +72,7 @@ "lodash.debounce": "^4.0.8", "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", - "n8n-workflow": "~0.64.0", + "n8n-workflow": "~0.65.0", "node-sass": "^4.12.0", "normalize-wheel": "^1.0.1", "prismjs": "^1.17.1", diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 6151d7e37e..8df78dabc5 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -353,6 +353,7 @@ export interface IExecutionsSummary { finished?: boolean; retryOf?: string; retrySuccessId?: string; + waitTill?: Date; startedAt: Date; stoppedAt?: Date; workflowId: string; diff --git a/packages/editor-ui/src/components/DuplicateWorkflowDialog.vue b/packages/editor-ui/src/components/DuplicateWorkflowDialog.vue index 839e00efac..e76fe0bf30 100644 --- a/packages/editor-ui/src/components/DuplicateWorkflowDialog.vue +++ b/packages/editor-ui/src/components/DuplicateWorkflowDialog.vue @@ -4,7 +4,7 @@ :eventBus="modalBus" @enter="save" size="sm" - title="Duplicate Workflow" + title="Duplicate Workflow" >