n8n/packages/cli/src/WorkflowExecuteAdditionalData.ts

799 lines
29 KiB
TypeScript
Raw Normal View History

2019-06-23 03:35:23 -07:00
import {
ActiveExecutions,
CredentialsHelper,
2019-06-23 03:35:23 -07:00
Db,
ExternalHooks,
2019-06-23 03:35:23 -07:00
IExecutionDb,
IExecutionFlattedDb,
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
IExecutionResponse,
2019-06-23 03:35:23 -07:00
IPushDataExecutionFinished,
IWorkflowBase,
IWorkflowExecuteProcess,
IWorkflowExecutionDataProcess,
2019-12-19 14:07:55 -08:00
NodeTypes,
2019-06-23 03:35:23 -07:00
Push,
ResponseHelper,
WebhookHelpers,
WorkflowCredentials,
2019-06-23 03:35:23 -07:00
WorkflowHelpers,
} from './';
import {
UserSettings,
2019-12-19 14:07:55 -08:00
WorkflowExecute,
} from 'n8n-core';
2019-06-23 03:35:23 -07:00
import {
IDataObject,
2019-12-19 14:07:55 -08:00
IExecuteData,
IExecuteWorkflowInfo,
2019-12-19 14:07:55 -08:00
INode,
INodeExecutionData,
2020-10-22 06:46:03 -07:00
INodeParameters,
2019-06-23 03:35:23 -07:00
IRun,
2019-12-19 14:07:55 -08:00
IRunExecutionData,
2019-06-23 03:35:23 -07:00
ITaskData,
IWorkflowCredentials,
2019-06-23 03:35:23 -07:00
IWorkflowExecuteAdditionalData,
IWorkflowExecuteHooks,
2019-12-19 14:07:55 -08:00
IWorkflowHooksOptionalParameters,
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
LoggerProxy as Logger,
2019-12-19 14:07:55 -08:00
Workflow,
2019-06-23 03:35:23 -07:00
WorkflowExecuteMode,
2019-12-19 14:07:55 -08:00
WorkflowHooks,
2019-06-23 03:35:23 -07:00
} from 'n8n-workflow';
import * as config from '../config';
2019-06-23 03:35:23 -07:00
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
import { LessThanOrEqual } from 'typeorm';
const ERROR_TRIGGER_TYPE = config.get('nodes.errorTriggerType') as string;
2019-06-23 03:35:23 -07:00
/**
* Checks if there was an error and if errorWorkflow or a trigger is defined. If so it collects
2019-06-23 03:35:23 -07:00
* all the data and executes it
*
* @param {IWorkflowBase} workflowData The workflow which got executed
* @param {IRun} fullRunData The run which produced the error
2019-12-19 14:07:55 -08:00
* @param {WorkflowExecuteMode} mode The mode in which the workflow got started in
2019-06-23 03:35:23 -07:00
* @param {string} [executionId] The id the execution got saved as
*/
function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mode: WorkflowExecuteMode, executionId?: string, retryOf?: string): void {
// Check if there was an error and if so if an errorWorkflow or a trigger is set
let pastExecutionUrl: string | undefined = undefined;
if (executionId !== undefined) {
pastExecutionUrl = `${WebhookHelpers.getWebhookBaseUrl()}execution/${executionId}`;
}
if (fullRunData.data.resultData.error !== undefined) {
2019-06-23 03:35:23 -07:00
const workflowErrorData = {
execution: {
id: executionId,
url: pastExecutionUrl,
2019-06-23 03:35:23 -07:00
error: fullRunData.data.resultData.error,
lastNodeExecuted: fullRunData.data.resultData.lastNodeExecuted!,
mode,
retryOf,
2019-06-23 03:35:23 -07:00
},
workflow: {
id: workflowData.id !== undefined ? workflowData.id.toString() as string : undefined,
name: workflowData.name,
2020-10-22 06:46:03 -07:00
},
2019-06-23 03:35:23 -07:00
};
2019-06-23 03:35:23 -07:00
// Run the error workflow
// To avoid an infinite loop do not run the error workflow again if the error-workflow itself failed and it is its own error-workflow.
2020-12-12 11:59:17 -08:00
if (workflowData.settings !== undefined && workflowData.settings.errorWorkflow && !(mode === 'error' && workflowData.id && workflowData.settings.errorWorkflow.toString() === workflowData.id.toString())) {
2021-05-05 13:18:14 -07:00
Logger.verbose(`Start external error workflow`, { executionId, errorWorkflowId: workflowData.settings.errorWorkflow.toString(), workflowId: workflowData.id });
// If a specific error workflow is set run only that one
WorkflowHelpers.executeErrorWorkflow(workflowData.settings.errorWorkflow as string, workflowErrorData);
} else if (mode !== 'error' && workflowData.id !== undefined && workflowData.nodes.some((node) => node.type === ERROR_TRIGGER_TYPE)) {
2021-05-05 13:18:14 -07:00
Logger.verbose(`Start internal error workflow`, { executionId, workflowId: workflowData.id });
// If the workflow contains
WorkflowHelpers.executeErrorWorkflow(workflowData.id.toString(), workflowErrorData);
}
2019-06-23 03:35:23 -07:00
}
}
/**
* Prunes Saved Execution which are older than configured.
* Throttled to be executed just once in configured timeframe.
*
*/
let throttling = false;
function pruneExecutionData(): void {
if (!throttling) {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.verbose('Pruning execution data from database');
throttling = true;
const timeout = config.get('executions.pruneDataTimeout') as number; // in seconds
const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h
const date = new Date(); // today
date.setHours(date.getHours() - maxAge);
// throttle just on success to allow for self healing on failure
Db.collections.Execution!.delete({ stoppedAt: LessThanOrEqual(date.toISOString()) })
.then(data =>
setTimeout(() => {
throttling = false;
}, timeout * 1000)
).catch(err => throttling = false);
}
}
2019-06-23 03:35:23 -07:00
/**
2019-12-19 14:07:55 -08:00
* Returns hook functions to push data to Editor-UI
*
* @returns {IWorkflowExecuteHooks}
*/
2019-12-19 14:07:55 -08:00
function hookFunctionsPush(): IWorkflowExecuteHooks {
2019-06-23 03:35:23 -07:00
return {
nodeExecuteBefore: [
2019-12-19 14:07:55 -08:00
async function (this: WorkflowHooks, nodeName: string): Promise<void> {
// Push data to session which started workflow before each
// node which starts rendering
2019-12-19 14:07:55 -08:00
if (this.sessionId === undefined) {
2019-06-23 03:35:23 -07:00
return;
}
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { executionId: this.executionId, sessionId: this.sessionId, workflowId: this.workflowData.id });
2019-06-23 03:35:23 -07:00
2019-08-28 06:28:47 -07:00
const pushInstance = Push.getInstance();
pushInstance.send('nodeExecuteBefore', {
2019-12-19 14:07:55 -08:00
executionId: this.executionId,
2019-06-23 03:35:23 -07:00
nodeName,
2019-12-19 14:07:55 -08:00
}, this.sessionId);
2019-06-23 03:35:23 -07:00
},
],
nodeExecuteAfter: [
2019-12-19 14:07:55 -08:00
async function (this: WorkflowHooks, nodeName: string, data: ITaskData): Promise<void> {
// Push data to session which started workflow after each rendered node
2019-12-19 14:07:55 -08:00
if (this.sessionId === undefined) {
2019-06-23 03:35:23 -07:00
return;
}
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Executing hook on node "${nodeName}" (hookFunctionsPush)`, { executionId: this.executionId, sessionId: this.sessionId, workflowId: this.workflowData.id });
2019-06-23 03:35:23 -07:00
2019-08-28 06:28:47 -07:00
const pushInstance = Push.getInstance();
pushInstance.send('nodeExecuteAfter', {
2019-12-19 14:07:55 -08:00
executionId: this.executionId,
2019-06-23 03:35:23 -07:00
nodeName,
data,
2019-12-19 14:07:55 -08:00
}, this.sessionId);
2019-06-23 03:35:23 -07:00
},
],
workflowExecuteBefore: [
2019-12-19 14:07:55 -08:00
async function (this: WorkflowHooks): Promise<void> {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Executing hook (hookFunctionsPush)`, { executionId: this.executionId, sessionId: this.sessionId, workflowId: this.workflowData.id });
// Push data to session which started the workflow
if (this.sessionId === undefined) {
return;
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
}
const pushInstance = Push.getInstance();
pushInstance.send('executionStarted', {
executionId: this.executionId,
mode: this.mode,
startedAt: new Date(),
retryOf: this.retryOf,
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
workflowId: this.workflowData.id, sessionId: this.sessionId as string,
workflowName: this.workflowData.name,
}, this.sessionId);
2020-10-22 06:46:03 -07:00
},
],
2019-06-23 03:35:23 -07:00
workflowExecuteAfter: [
2019-12-19 14:07:55 -08:00
async function (this: WorkflowHooks, fullRunData: IRun, newStaticData: IDataObject): Promise<void> {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Executing hook (hookFunctionsPush)`, { executionId: this.executionId, sessionId: this.sessionId, workflowId: this.workflowData.id });
// Push data to session which started the workflow
if (this.sessionId === undefined) {
return;
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
}
// Clone the object except the runData. That one is not supposed
// to be send. Because that data got send piece by piece after
// each node which finished executing
const pushRunData = {
...fullRunData,
data: {
...fullRunData.data,
resultData: {
...fullRunData.data.resultData,
runData: {},
},
},
};
// Push data to editor-ui once workflow finished
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Save execution progress to database for execution ID ${this.executionId} `, { executionId: this.executionId, workflowId: this.workflowData.id });
// TODO: Look at this again
const sendData: IPushDataExecutionFinished = {
executionId: this.executionId,
data: pushRunData,
retryOf: this.retryOf,
};
const pushInstance = Push.getInstance();
pushInstance.send('executionFinished', sendData, this.sessionId);
2019-12-19 14:07:55 -08:00
},
2020-10-22 06:46:03 -07:00
],
2019-12-19 14:07:55 -08:00
};
}
export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowExecuteHooks {
const externalHooks = ExternalHooks();
return {
workflowExecuteBefore: [
async function (this: WorkflowHooks, workflow: Workflow): Promise<void> {
await externalHooks.run('workflow.preExecute', [workflow, this.mode]);
},
],
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
nodeExecuteAfter: [
async function (nodeName: string, data: ITaskData, executionData: IRunExecutionData): Promise<void> {
if (this.workflowData.settings !== undefined) {
if (this.workflowData.settings.saveExecutionProgress === false) {
return;
} else if (this.workflowData.settings.saveExecutionProgress !== true && !config.get('executions.saveExecutionProgress') as boolean) {
return;
}
} else if (!config.get('executions.saveExecutionProgress') as boolean) {
return;
}
try {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Save execution progress to database for execution ID ${this.executionId} `, { executionId: this.executionId, nodeName });
const execution = await Db.collections.Execution!.findOne(this.executionId);
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (execution === undefined) {
// Something went badly wrong if this happens.
// This check is here mostly to make typescript happy.
return undefined;
}
const fullExecutionData: IExecutionResponse = ResponseHelper.unflattenExecutionData(execution);
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (fullExecutionData.finished) {
// We already received ´workflowExecuteAfter´ webhook, so this is just an async call
// that was left behind. We skip saving because the other call should have saved everything
// so this one is safe to ignore
return;
}
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (fullExecutionData.data === undefined) {
fullExecutionData.data = {
startData: {
},
resultData: {
runData: {},
},
executionData: {
contextData: {},
nodeExecutionStack: [],
waitingExecution: {},
},
};
}
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (Array.isArray(fullExecutionData.data.resultData.runData[nodeName])) {
// Append data if array exists
fullExecutionData.data.resultData.runData[nodeName].push(data);
} else {
// Initialize array and save data
fullExecutionData.data.resultData.runData[nodeName] = [data];
}
fullExecutionData.data.executionData = executionData.executionData;
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
// Set last executed node so that it may resume on failure
fullExecutionData.data.resultData.lastNodeExecuted = nodeName;
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
const flattenedExecutionData = ResponseHelper.flattenExecutionData(fullExecutionData);
await Db.collections.Execution!.update(this.executionId, flattenedExecutionData as IExecutionFlattedDb);
} catch (err) {
// TODO: Improve in the future!
// Errors here might happen because of database access
// For busy machines, we may get "Database is locked" errors.
// We do this to prevent crashes and executions ending in `unknown` state.
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.error(`Failed saving execution progress to database for execution ID ${this.executionId} (hookFunctionsPreExecute, nodeExecuteAfter)`, { ...err, executionId: this.executionId, sessionId: this.sessionId, workflowId: this.workflowData.id });
}
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
},
],
};
}
2019-12-19 14:07:55 -08:00
/**
* Returns hook functions to save workflow execution and call error workflow
*
* @returns {IWorkflowExecuteHooks}
*/
function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
return {
nodeExecuteBefore: [],
nodeExecuteAfter: [],
workflowExecuteBefore: [],
workflowExecuteAfter: [
async function (this: WorkflowHooks, fullRunData: IRun, newStaticData: IDataObject): Promise<void> {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.debug(`Executing hook (hookFunctionsSave)`, { executionId: this.executionId, workflowId: this.workflowData.id });
2019-12-19 14:07:55 -08:00
// Prune old execution data
if (config.get('executions.pruneData')) {
pruneExecutionData();
}
2019-12-19 14:07:55 -08:00
const isManualMode = [this.mode, parentProcessMode].includes('manual');
2019-06-23 03:35:23 -07:00
try {
2019-12-19 14:07:55 -08:00
if (!isManualMode && WorkflowHelpers.isWorkflowIdValid(this.workflowData.id as string) === true && newStaticData) {
// Workflow is saved so update in database
try {
2019-12-19 14:07:55 -08:00
await WorkflowHelpers.saveStaticDataById(this.workflowData.id as string, newStaticData);
} catch (e) {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.error(`There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (hookFunctionsSave)`, { executionId: this.executionId, workflowId: this.workflowData.id });
}
}
2019-06-23 03:35:23 -07:00
let saveManualExecutions = config.get('executions.saveDataManualExecutions') as boolean;
2019-12-19 14:07:55 -08:00
if (this.workflowData.settings !== undefined && this.workflowData.settings.saveManualExecutions !== undefined) {
2019-06-23 03:35:23 -07:00
// Apply to workflow override
2019-12-19 14:07:55 -08:00
saveManualExecutions = this.workflowData.settings.saveManualExecutions as boolean;
2019-06-23 03:35:23 -07:00
}
2019-12-19 14:07:55 -08:00
if (isManualMode && saveManualExecutions === false) {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
// Data is always saved, so we remove from database
await Db.collections.Execution!.delete(this.executionId);
return;
}
// Check config to know if execution should be saved or not
let saveDataErrorExecution = config.get('executions.saveDataOnError') as string;
let saveDataSuccessExecution = config.get('executions.saveDataOnSuccess') as string;
2019-12-19 14:07:55 -08:00
if (this.workflowData.settings !== undefined) {
saveDataErrorExecution = (this.workflowData.settings.saveDataErrorExecution as string) || saveDataErrorExecution;
saveDataSuccessExecution = (this.workflowData.settings.saveDataSuccessExecution as string) || saveDataSuccessExecution;
}
2019-06-23 03:35:23 -07:00
const workflowDidSucceed = !fullRunData.data.resultData.error;
if (workflowDidSucceed === true && saveDataSuccessExecution === 'none' ||
workflowDidSucceed === false && saveDataErrorExecution === 'none'
) {
2019-12-19 14:07:55 -08:00
if (!isManualMode) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf);
}
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
// Data is always saved, so we remove from database
await Db.collections.Execution!.delete(this.executionId);
2019-06-23 03:35:23 -07:00
return;
}
const fullExecutionData: IExecutionDb = {
data: fullRunData.data,
mode: fullRunData.mode,
finished: fullRunData.finished ? fullRunData.finished : false,
startedAt: fullRunData.startedAt,
stoppedAt: fullRunData.stoppedAt,
2019-12-19 14:07:55 -08:00
workflowData: this.workflowData,
2019-06-23 03:35:23 -07:00
};
2019-12-19 14:07:55 -08:00
if (this.retryOf !== undefined) {
fullExecutionData.retryOf = this.retryOf.toString();
2019-06-23 03:35:23 -07:00
}
2019-12-19 14:07:55 -08:00
if (this.workflowData.id !== undefined && WorkflowHelpers.isWorkflowIdValid(this.workflowData.id.toString()) === true) {
fullExecutionData.workflowId = this.workflowData.id.toString();
2019-06-23 03:35:23 -07:00
}
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
// Leave log message before flatten as that operation increased memory usage a lot and the chance of a crash is highest here
Logger.debug(`Save execution data to database for execution ID ${this.executionId}`, { executionId: this.executionId, workflowId: this.workflowData.id });
2019-06-23 03:35:23 -07:00
const executionData = ResponseHelper.flattenExecutionData(fullExecutionData);
// Save the Execution in DB
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
await Db.collections.Execution!.update(this.executionId, executionData as IExecutionFlattedDb);
2019-06-23 03:35:23 -07:00
2019-12-19 14:07:55 -08:00
if (fullRunData.finished === true && this.retryOf !== undefined) {
2019-06-23 03:35:23 -07:00
// If the retry was successful save the reference it on the original execution
// await Db.collections.Execution!.save(executionData as IExecutionFlattedDb);
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: this.executionId });
2019-06-23 03:35:23 -07:00
}
2019-12-19 14:07:55 -08:00
if (!isManualMode) {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, this.executionId, this.retryOf);
2019-12-19 14:07:55 -08:00
}
2019-06-23 03:35:23 -07:00
} catch (error) {
2019-12-19 14:07:55 -08:00
if (!isManualMode) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf);
}
2019-06-23 03:35:23 -07:00
}
},
2020-10-22 06:46:03 -07:00
],
2019-06-23 03:35:23 -07:00
};
2019-12-19 14:07:55 -08:00
}
/**
* Returns hook functions to save workflow execution and call error workflow
* for running with queues. Manual executions should never run on queues as
* they are always executed in the main process.
*
* @returns {IWorkflowExecuteHooks}
*/
function hookFunctionsSaveWorker(): IWorkflowExecuteHooks {
return {
nodeExecuteBefore: [],
nodeExecuteAfter: [],
workflowExecuteBefore: [],
workflowExecuteAfter: [
async function (this: WorkflowHooks, fullRunData: IRun, newStaticData: IDataObject): Promise<void> {
try {
if (WorkflowHelpers.isWorkflowIdValid(this.workflowData.id as string) === true && newStaticData) {
// Workflow is saved so update in database
try {
await WorkflowHelpers.saveStaticDataById(this.workflowData.id as string, newStaticData);
} catch (e) {
:sparkles: Added logging to n8n (#1381) * Added logging to n8n This commit adds logging to n8n using the Winston library. For now, this commit only allows logging to console (default behavior) or file (need to pass in config via environment variables). Other logging methods can be further implemented using hooks. These were skipped for now as it would require adding more dependencies. Logging level is notice by default, meaning no additional messages would be displayed at the moment. Logging level can be set to info or debug as well to enrich the generated logs. The ILogger interface was added to the workflow project as it would make it available for all other projects but the implementation was done on the cli project. * Lint fixes and logging level naming. Also fixed the way we use the logger as it was not working previously * Improvements to logging framework Using appropriate single quotes Improving the way the logger is declared * Improved naming for Log Types * Removed logger global variable, replacing it by a proxy * Add logging to CLI commands * Remove unused GenericHelpers * Changed back some messages to console instead of logger and added npm shortcuts for worker and webhook * Fix typos * Adding basic file rotation to logs as suggested by @mutdmour * Fixed linting issues * Correcting comment to correctly reflect space usage * Added settings for log files rotation * Correcting config type from String to Number * Changed default file settings to number To reflect previous changes to the type * Changed the way log messages are added to be called statically. Also minor naming improvements * Applying latest corrections sent by @ivov * :zap: Some logging improvements * Saving logs to a folder inside n8n home instead of root * Fixed broken tests and linting * Changed some log messages to improve formatting * Adding quotes to names on log messages * Added execution and session IDs to logs. Also removed unnecessary line breaks * :zap: Added file caller to log messages (#1657) This is done using callsites library which already existed in the project as another library's dependency. So in fact it does not add any new dependency. * Adding logs to help debug Salesforce node * :zap: Add function name to logs and add more logs * :zap: Improve some error messages * :zap: Improve some more log messages * :zap: Rename logging env variables to match others Co-authored-by: dali <servfrdali@yahoo.fr> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-05-01 20:43:01 -07:00
Logger.error(`There was a problem saving the workflow with id "${this.workflowData.id}" to save changed staticData: "${e.message}" (workflowExecuteAfter)`, { sessionId: this.sessionId, workflowId: this.workflowData.id });
}
}
const workflowDidSucceed = !fullRunData.data.resultData.error;
if (workflowDidSucceed === false) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf);
}
const fullExecutionData: IExecutionDb = {
data: fullRunData.data,
mode: fullRunData.mode,
finished: fullRunData.finished ? fullRunData.finished : false,
startedAt: fullRunData.startedAt,
stoppedAt: fullRunData.stoppedAt,
workflowData: this.workflowData,
};
if (this.retryOf !== undefined) {
fullExecutionData.retryOf = this.retryOf.toString();
}
if (this.workflowData.id !== undefined && WorkflowHelpers.isWorkflowIdValid(this.workflowData.id.toString()) === true) {
fullExecutionData.workflowId = this.workflowData.id.toString();
}
const executionData = ResponseHelper.flattenExecutionData(fullExecutionData);
// Save the Execution in DB
await Db.collections.Execution!.update(this.executionId, executionData as IExecutionFlattedDb);
if (fullRunData.finished === true && this.retryOf !== undefined) {
// If the retry was successful save the reference it on the original execution
await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: this.executionId });
}
} catch (error) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, undefined, this.retryOf);
}
},
],
};
}
export async function getRunData(workflowData: IWorkflowBase, inputData?: INodeExecutionData[]): Promise<IWorkflowExecutionDataProcess> {
2019-12-19 14:07:55 -08:00
const mode = 'integrated';
// Find Start-Node
const requiredNodeTypes = ['n8n-nodes-base.start'];
let startNode: INode | undefined;
for (const node of workflowData!.nodes) {
if (requiredNodeTypes.includes(node.type)) {
startNode = node;
break;
}
}
if (startNode === undefined) {
// If the workflow does not contain a start-node we can not know what
// should be executed and with what data to start.
throw new Error(`The workflow does not contain a "Start" node and can so not be executed.`);
}
// Always start with empty data if no inputData got supplied
inputData = inputData || [
{
2020-10-22 06:46:03 -07:00
json: {},
},
2019-12-19 14:07:55 -08:00
];
// Initialize the incoming data
const nodeExecutionStack: IExecuteData[] = [];
nodeExecutionStack.push(
{
node: startNode,
data: {
main: [inputData],
},
2020-10-22 06:46:03 -07:00
}
2019-12-19 14:07:55 -08:00
);
const runExecutionData: IRunExecutionData = {
startData: {
},
resultData: {
runData: {},
},
executionData: {
contextData: {},
nodeExecutionStack,
waitingExecution: {},
},
};
// Get the needed credentials for the current workflow as they will differ to the ones of the
// calling workflow.
const credentials = await WorkflowCredentials(workflowData!.nodes);
const runData: IWorkflowExecutionDataProcess = {
credentials,
executionMode: mode,
executionData: runExecutionData,
// @ts-ignore
workflowData,
};
return runData;
}
export async function getWorkflowData(workflowInfo: IExecuteWorkflowInfo): Promise<IWorkflowBase> {
if (workflowInfo.id === undefined && workflowInfo.code === undefined) {
throw new Error(`No information about the workflow to execute found. Please provide either the "id" or "code"!`);
}
if (Db.collections!.Workflow === null) {
// The first time executeWorkflow gets called the Database has
// to get initialized first
await Db.init();
}
let workflowData: IWorkflowBase | undefined;
if (workflowInfo.id !== undefined) {
workflowData = await Db.collections!.Workflow!.findOne(workflowInfo.id);
if (workflowData === undefined) {
throw new Error(`The workflow with the id "${workflowInfo.id}" does not exist.`);
}
} else {
workflowData = workflowInfo.code;
}
return workflowData!;
}
/**
* Executes the workflow with the given ID
*
* @export
* @param {string} workflowId The id of the workflow to execute
* @param {IWorkflowExecuteAdditionalData} additionalData
* @param {INodeExecutionData[]} [inputData]
* @returns {(Promise<Array<INodeExecutionData[] | null>>)}
*/
export async function executeWorkflow(workflowInfo: IExecuteWorkflowInfo, additionalData: IWorkflowExecuteAdditionalData, inputData?: INodeExecutionData[], parentExecutionId?: string, loadedWorkflowData?: IWorkflowBase, loadedRunData?: IWorkflowExecutionDataProcess): Promise<Array<INodeExecutionData[] | null> | IWorkflowExecuteProcess> {
const externalHooks = ExternalHooks();
await externalHooks.init();
const nodeTypes = NodeTypes();
const workflowData = loadedWorkflowData !== undefined ? loadedWorkflowData : await getWorkflowData(workflowInfo);
const workflowName = workflowData ? workflowData.name : undefined;
const workflow = new Workflow({ id: workflowInfo.id, name: workflowName, nodes: workflowData!.nodes, connections: workflowData!.connections, active: workflowData!.active, nodeTypes, staticData: workflowData!.staticData });
const runData = loadedRunData !== undefined ? loadedRunData : await getRunData(workflowData, inputData);
let executionId;
if (parentExecutionId !== undefined) {
executionId = parentExecutionId;
} else {
executionId = parentExecutionId !== undefined ? parentExecutionId : await ActiveExecutions.getInstance().add(runData);
}
const runExecutionData = runData.executionData as IRunExecutionData;
// Get the needed credentials for the current workflow as they will differ to the ones of the
// calling workflow.
const credentials = await WorkflowCredentials(workflowData!.nodes);
// Create new additionalData to have different workflow loaded and to call
// different webooks
const additionalDataIntegrated = await getBase(credentials);
additionalDataIntegrated.hooks = getWorkflowHooksIntegrated(runData.executionMode, executionId, workflowData!, { parentProcessMode: additionalData.hooks!.mode });
// Make sure we pass on the original executeWorkflow function we received
// This one already contains changes to talk to parent process
// and get executionID from `activeExecutions` running on main process
additionalDataIntegrated.executeWorkflow = additionalData.executeWorkflow;
let subworkflowTimeout = additionalData.executionTimeoutTimestamp;
if (workflowData.settings?.executionTimeout !== undefined && workflowData.settings.executionTimeout > 0) {
// We might have received a max timeout timestamp from the parent workflow
// If we did, then we get the minimum time between the two timeouts
// If no timeout was given from the parent, then we use our timeout.
subworkflowTimeout = Math.min(additionalData.executionTimeoutTimestamp || Number.MAX_SAFE_INTEGER, Date.now() + (workflowData.settings.executionTimeout as number * 1000));
}
additionalDataIntegrated.executionTimeoutTimestamp = subworkflowTimeout;
2019-12-19 14:07:55 -08:00
// Execute the workflow
const workflowExecute = new WorkflowExecute(additionalDataIntegrated, runData.executionMode, runExecutionData);
if (parentExecutionId !== undefined) {
// Must be changed to become typed
return {
startedAt: new Date(),
workflow,
workflowExecute,
};
}
2019-12-19 14:07:55 -08:00
const data = await workflowExecute.processRunExecutionData(workflow);
await externalHooks.run('workflow.postExecute', [data, workflowData]);
2019-12-19 14:07:55 -08:00
if (data.finished === true) {
// Workflow did finish successfully
await ActiveExecutions.getInstance().remove(executionId, data);
const returnData = WorkflowHelpers.getDataLastExecutedNodeData(data);
return returnData!.data!.main;
2019-12-19 14:07:55 -08:00
} else {
await ActiveExecutions.getInstance().remove(executionId, data);
2019-12-19 14:07:55 -08:00
// Workflow did fail
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
const { error } = data.data.resultData;
throw {
...error,
stack: error!.stack,
};
2019-12-19 14:07:55 -08:00
}
}
2019-06-23 03:35:23 -07:00
export function sendMessageToUI(source: string, message: string) {
if (this.sessionId === undefined) {
return;
}
// Push data to session which started workflow
try {
const pushInstance = Push.getInstance();
pushInstance.send('sendConsoleMessage', {
source: `Node: "${source}"`,
message,
}, this.sessionId);
} catch (error) {
Logger.warn(`There was a problem sending messsage to UI: ${error.message}`);
}
}
/**
* Returns the base additional data without webhooks
*
* @export
* @param {IWorkflowCredentials} credentials
* @param {INodeParameters} currentNodeParameters
* @returns {Promise<IWorkflowExecuteAdditionalData>}
*/
export async function getBase(credentials: IWorkflowCredentials, currentNodeParameters?: INodeParameters, executionTimeoutTimestamp?: number): Promise<IWorkflowExecuteAdditionalData> {
2019-06-23 03:35:23 -07:00
const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
const timezone = config.get('generic.timezone') as string;
const webhookBaseUrl = urlBaseWebhook + config.get('endpoints.webhook') as string;
const webhookTestBaseUrl = urlBaseWebhook + config.get('endpoints.webhookTest') as string;
2019-06-23 03:35:23 -07:00
const encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
throw new Error('No encryption key got found to decrypt the credentials!');
}
return {
credentials,
credentialsHelper: new CredentialsHelper(credentials, encryptionKey),
2019-06-23 03:35:23 -07:00
encryptionKey,
2019-12-19 14:07:55 -08:00
executeWorkflow,
restApiUrl: urlBaseWebhook + config.get('endpoints.rest') as string,
2019-06-23 03:35:23 -07:00
timezone,
webhookBaseUrl,
webhookTestBaseUrl,
2019-10-20 12:42:34 -07:00
currentNodeParameters,
executionTimeoutTimestamp,
2019-06-23 03:35:23 -07:00
};
}
/**
2019-12-19 14:07:55 -08:00
* Returns WorkflowHooks instance for running integrated workflows
* (Workflows which get started inside of another workflow)
*/
export function getWorkflowHooksIntegrated(mode: WorkflowExecuteMode, executionId: string, workflowData: IWorkflowBase, optionalParameters?: IWorkflowHooksOptionalParameters): WorkflowHooks {
optionalParameters = optionalParameters || {};
const hookFunctions = hookFunctionsSave(optionalParameters.parentProcessMode);
const preExecuteFunctions = hookFunctionsPreExecute(optionalParameters.parentProcessMode);
for (const key of Object.keys(preExecuteFunctions)) {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (hookFunctions[key] === undefined) {
hookFunctions[key] = [];
}
hookFunctions[key]!.push.apply(hookFunctions[key], preExecuteFunctions[key]);
}
2019-12-19 14:07:55 -08:00
return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters);
}
/**
* Returns WorkflowHooks instance for running integrated workflows
* (Workflows which get started inside of another workflow)
*/
export function getWorkflowHooksWorkerExecuter(mode: WorkflowExecuteMode, executionId: string, workflowData: IWorkflowBase, optionalParameters?: IWorkflowHooksOptionalParameters): WorkflowHooks {
optionalParameters = optionalParameters || {};
const hookFunctions = hookFunctionsSaveWorker();
const preExecuteFunctions = hookFunctionsPreExecute(optionalParameters.parentProcessMode);
for (const key of Object.keys(preExecuteFunctions)) {
if (hookFunctions[key] === undefined) {
hookFunctions[key] = [];
}
hookFunctions[key]!.push.apply(hookFunctions[key], preExecuteFunctions[key]);
}
return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters);
}
2019-12-19 14:07:55 -08:00
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
/**
* Returns WorkflowHooks instance for main process if workflow runs via worker
*/
export function getWorkflowHooksWorkerMain(mode: WorkflowExecuteMode, executionId: string, workflowData: IWorkflowBase, optionalParameters?: IWorkflowHooksOptionalParameters): WorkflowHooks {
optionalParameters = optionalParameters || {};
const hookFunctions = hookFunctionsPush();
const preExecuteFunctions = hookFunctionsPreExecute(optionalParameters.parentProcessMode);
for (const key of Object.keys(preExecuteFunctions)) {
if (hookFunctions[key] === undefined) {
hookFunctions[key] = [];
}
hookFunctions[key]!.push.apply(hookFunctions[key], preExecuteFunctions[key]);
}
// When running with worker mode, main process executes
// Only workflowExecuteBefore + workflowExecuteAfter
// So to avoid confusion, we are removing other hooks.
hookFunctions.nodeExecuteBefore = [];
hookFunctions.nodeExecuteAfter = [];
return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters);
}
2019-12-19 14:07:55 -08:00
/**
* Returns WorkflowHooks instance for running the main workflow
*
* @export
* @param {IWorkflowExecutionDataProcess} data
* @param {string} executionId
2019-12-19 14:07:55 -08:00
* @returns {WorkflowHooks}
*/
export function getWorkflowHooksMain(data: IWorkflowExecutionDataProcess, executionId: string, isMainProcess = false): WorkflowHooks {
2019-12-19 14:07:55 -08:00
const hookFunctions = hookFunctionsSave();
const pushFunctions = hookFunctionsPush();
for (const key of Object.keys(pushFunctions)) {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (hookFunctions[key] === undefined) {
hookFunctions[key] = [];
}
2019-12-19 14:07:55 -08:00
hookFunctions[key]!.push.apply(hookFunctions[key], pushFunctions[key]);
}
if (isMainProcess) {
const preExecuteFunctions = hookFunctionsPreExecute();
for (const key of Object.keys(preExecuteFunctions)) {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (hookFunctions[key] === undefined) {
hookFunctions[key] = [];
}
hookFunctions[key]!.push.apply(hookFunctions[key], preExecuteFunctions[key]);
}
}
return new WorkflowHooks(hookFunctions, data.executionMode, executionId, data.workflowData, { sessionId: data.sessionId, retryOf: data.retryOf as string });
}