n8n/packages/cli/src/ActiveExecutions.ts

208 lines
5.5 KiB
TypeScript
Raw Normal View History

2019-06-23 03:35:23 -07:00
import {
IRun,
} from 'n8n-workflow';
import {
createDeferredPromise,
} from 'n8n-core';
import {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
Db,
IExecutingWorkflowData,
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
IExecutionDb,
IExecutionFlattedDb,
2020-10-22 06:46:03 -07:00
IExecutionsCurrentSummary,
IWorkflowExecutionDataProcess,
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
ResponseHelper,
WorkflowHelpers,
2019-06-23 03:35:23 -07:00
} from '.';
import { ChildProcess } from 'child_process';
import * as PCancelable from 'p-cancelable';
2019-06-23 03:35:23 -07:00
export class ActiveExecutions {
private activeExecutions: {
[index: string]: IExecutingWorkflowData;
} = {};
/**
* Add a new active execution
*
* @param {ChildProcess} process
* @param {IWorkflowExecutionDataProcess} executionData
2019-06-23 03:35:23 -07:00
* @returns {string}
* @memberof ActiveExecutions
*/
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
async add(executionData: IWorkflowExecutionDataProcess, process?: ChildProcess): Promise<string> {
const fullExecutionData: IExecutionDb = {
data: executionData.executionData!,
mode: executionData.executionMode,
finished: false,
startedAt: new Date(),
workflowData: executionData.workflowData,
};
if (executionData.retryOf !== undefined) {
fullExecutionData.retryOf = executionData.retryOf.toString();
}
if (executionData.workflowData.id !== undefined && WorkflowHelpers.isWorkflowIdValid(executionData.workflowData.id.toString()) === true) {
fullExecutionData.workflowId = executionData.workflowData.id.toString();
}
:sparkles: Add tagging of workflows (#1647) * clean up dropdown * clean up focusoncreate * :zap: Ignore mistaken ID in POST /workflows * :zap: Fix undefined tag ID in PATCH /workflows * :zap: Shorten response for POST /tags * remove scss mixins * clean up imports * :zap: Implement validation with class-validator * address ivan's comments * implement modals * Fix lint issues * fix disabling shortcuts * fix focus issues * fix focus issues * fix focus issues with modal * fix linting issues * use dispatch * use constants for modal keys * fix focus * fix lint issues * remove unused prop * add modal root * fix lint issues * remove unused methods * fix shortcut * remove max width * :zap: Fix duplicate entry error for pg and MySQL * update rename messaging * update order of buttons * fix firefox overflow on windows * fix dropdown height * :hammer: refactor tag crud controllers * 🧹 remove unused imports * use variable for number of items * fix dropdown spacing * :zap: Restore type to fix build * :zap: Fix post-refactor PATCH /workflows/:id * :zap: Fix PATCH /workflows/:id for zero tags * :zap: Fix usage count becoming stringified * address max's comments * fix filter spacing * fix blur bug * address most of ivan's comments * address tags type concern * remove defaults * :zap: return tag id as string * :hammer: add hooks to tag CUD operations * 🏎 simplify timestamp pruning * remove blur event * fix onblur bug * :zap: Fix fs import to fix build * address max's comments * implement responsive tag container * fix lint issues * Set default dates in entities * :shirt: Fix lint in migrations * update tag limits * address ivan's comments * remove rename, refactor header, implement new designs for save, remove responsive tag container * update styling * update styling * implement responsive tag container * implement header tags edit * implement header tags edit * fix lint issues * implement expandable input * minor fixes * minor fixes * use variable * rename save as * duplicate fixes * :zap: Implement unique workflow names * :zap: Create /workflows/new endpoint * minor edit fixes * lint fixes * style fixes * hook up saving name * hook up tags * clean up impl * fix dirty state bug * update limit * update notification messages * on click outside * fix minor bug with count * lint fixes * :zap: Add query string params to /workflows/new * handle minor edge cases * handle minor edge cases * handle minor bugs; fix firefox dropdown issue * Fix min width * apply tags only after api success * remove count fix * :construction: Adjust to new qs requirements * clean up workflow tags impl, fix tags delete bug * fix minor issue * fix minor spacing issue * disable wrap for ops * fix viewport root; save on click in dropdown * save button loading when saving name/tags * implement max width on tags container * implement cleaner create experience * disable edit while updating * codacy hex color * refactor tags container * fix clickability * fix workflow open and count * clean up structure * fix up lint issues * :zap: Create migrations for unique workflow names * fix button size * increase workflow name limit for larger screen * tslint fixes * disable responsiveness for workflow modal * rename event * change min width for tags * clean up pr * :zap: Adjust quotes in MySQL migration * :zap: Adjust quotes in Postgres migration * address max's comments on styles * remove success toasts * add hover mode to name * minor fixes * refactor name preview * fix name input not to jiggle * finish up name input * Fix up add tags * clean up param * clean up scss * fix resizing name * fix resizing name * fix resize bug * clean up edit spacing * ignore on esc * fix input bug * focus input on clear * build * fix up add tags clickablity * remove scrollbars * move into folders * clean up multiple patch req * remove padding top from edit * update tags on enter * build * rollout blur on enter behavior * rollout esc behavior * fix tags bug when duplicating tags * move key to reload tags * update header spacing * build * update hex case * refactor workflow title * remove unusued prop * keep focus on error, fix bug on error * Fix bug with name / tags toggle on error * impl creating new workflow name * :zap: Refactor endpoint per new guidelines * support naming endpoint * :zap: Refactor to support numeric suffixes * :shirt: Lint migrations for unique workflow names * :zap: Add migrations set default dates to indexes * fix connection push bug * :zap: Lowercase default workflow name * :zap: Add prefixes to set default dates migration * :zap: Fix indentation on default dates migrations * :zap: Add temp ts-ignore for unrelated change * :zap: Adjust default dates migration for MySQL Remove change to data column in credentials_entity, already covered by Omar's migration. Also, fix quotes from table prefix addition. * :zap: Adjust quotes in dates migration for PG * fix safari color bug * fix count bug * fix scroll bugs in dropdown * expand filter size * apply box-sizing to main header * update workflow names in executions to be wrapped by quotes * fix bug where key is same in dropdown * fix firefox bug * move up push connection session * :hammer: Remove mistakenly added nullable property * :fire: Remove unneeded index drop-create (PG) * :fire: Remove unneeded table copying * :zap: Merge dates migration with tags migration * :hammer: Refactor endpoint and make wf name env * dropdown colors in firefox * update colors to use variables * update thumb color * change error message * remove 100 char maximum * fix bug with saving tags dropdowns multiple times * update error message when no name * :zap: Update name missing toast message * :zap: Update workflow already exists message * disable saving for executions * fix bug causing modal to close * make tags in workflow open clickable * increase workflow limit to 3 * remove success notifications * update header spacing * escape tag names * update tag and table colors * remove tags from export * build * clean up push connection dependencies * address ben's comments * revert tags optional interface * address comments * update duplicate message * build * fix eol * add one more eol * :zap: Update comment * add hover style for workflow open, fix up font weight Co-authored-by: Mutasem <mutdmour@gmail.com> Co-authored-by: Iván Ovejero <ivov.src@gmail.com> Co-authored-by: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com>
2021-05-29 11:31:21 -07:00
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
const execution = ResponseHelper.flattenExecutionData(fullExecutionData);
// Save the Execution in DB
const executionResult = await Db.collections.Execution!.save(execution as IExecutionFlattedDb);
:sparkles: Add tagging of workflows (#1647) * clean up dropdown * clean up focusoncreate * :zap: Ignore mistaken ID in POST /workflows * :zap: Fix undefined tag ID in PATCH /workflows * :zap: Shorten response for POST /tags * remove scss mixins * clean up imports * :zap: Implement validation with class-validator * address ivan's comments * implement modals * Fix lint issues * fix disabling shortcuts * fix focus issues * fix focus issues * fix focus issues with modal * fix linting issues * use dispatch * use constants for modal keys * fix focus * fix lint issues * remove unused prop * add modal root * fix lint issues * remove unused methods * fix shortcut * remove max width * :zap: Fix duplicate entry error for pg and MySQL * update rename messaging * update order of buttons * fix firefox overflow on windows * fix dropdown height * :hammer: refactor tag crud controllers * 🧹 remove unused imports * use variable for number of items * fix dropdown spacing * :zap: Restore type to fix build * :zap: Fix post-refactor PATCH /workflows/:id * :zap: Fix PATCH /workflows/:id for zero tags * :zap: Fix usage count becoming stringified * address max's comments * fix filter spacing * fix blur bug * address most of ivan's comments * address tags type concern * remove defaults * :zap: return tag id as string * :hammer: add hooks to tag CUD operations * 🏎 simplify timestamp pruning * remove blur event * fix onblur bug * :zap: Fix fs import to fix build * address max's comments * implement responsive tag container * fix lint issues * Set default dates in entities * :shirt: Fix lint in migrations * update tag limits * address ivan's comments * remove rename, refactor header, implement new designs for save, remove responsive tag container * update styling * update styling * implement responsive tag container * implement header tags edit * implement header tags edit * fix lint issues * implement expandable input * minor fixes * minor fixes * use variable * rename save as * duplicate fixes * :zap: Implement unique workflow names * :zap: Create /workflows/new endpoint * minor edit fixes * lint fixes * style fixes * hook up saving name * hook up tags * clean up impl * fix dirty state bug * update limit * update notification messages * on click outside * fix minor bug with count * lint fixes * :zap: Add query string params to /workflows/new * handle minor edge cases * handle minor edge cases * handle minor bugs; fix firefox dropdown issue * Fix min width * apply tags only after api success * remove count fix * :construction: Adjust to new qs requirements * clean up workflow tags impl, fix tags delete bug * fix minor issue * fix minor spacing issue * disable wrap for ops * fix viewport root; save on click in dropdown * save button loading when saving name/tags * implement max width on tags container * implement cleaner create experience * disable edit while updating * codacy hex color * refactor tags container * fix clickability * fix workflow open and count * clean up structure * fix up lint issues * :zap: Create migrations for unique workflow names * fix button size * increase workflow name limit for larger screen * tslint fixes * disable responsiveness for workflow modal * rename event * change min width for tags * clean up pr * :zap: Adjust quotes in MySQL migration * :zap: Adjust quotes in Postgres migration * address max's comments on styles * remove success toasts * add hover mode to name * minor fixes * refactor name preview * fix name input not to jiggle * finish up name input * Fix up add tags * clean up param * clean up scss * fix resizing name * fix resizing name * fix resize bug * clean up edit spacing * ignore on esc * fix input bug * focus input on clear * build * fix up add tags clickablity * remove scrollbars * move into folders * clean up multiple patch req * remove padding top from edit * update tags on enter * build * rollout blur on enter behavior * rollout esc behavior * fix tags bug when duplicating tags * move key to reload tags * update header spacing * build * update hex case * refactor workflow title * remove unusued prop * keep focus on error, fix bug on error * Fix bug with name / tags toggle on error * impl creating new workflow name * :zap: Refactor endpoint per new guidelines * support naming endpoint * :zap: Refactor to support numeric suffixes * :shirt: Lint migrations for unique workflow names * :zap: Add migrations set default dates to indexes * fix connection push bug * :zap: Lowercase default workflow name * :zap: Add prefixes to set default dates migration * :zap: Fix indentation on default dates migrations * :zap: Add temp ts-ignore for unrelated change * :zap: Adjust default dates migration for MySQL Remove change to data column in credentials_entity, already covered by Omar's migration. Also, fix quotes from table prefix addition. * :zap: Adjust quotes in dates migration for PG * fix safari color bug * fix count bug * fix scroll bugs in dropdown * expand filter size * apply box-sizing to main header * update workflow names in executions to be wrapped by quotes * fix bug where key is same in dropdown * fix firefox bug * move up push connection session * :hammer: Remove mistakenly added nullable property * :fire: Remove unneeded index drop-create (PG) * :fire: Remove unneeded table copying * :zap: Merge dates migration with tags migration * :hammer: Refactor endpoint and make wf name env * dropdown colors in firefox * update colors to use variables * update thumb color * change error message * remove 100 char maximum * fix bug with saving tags dropdowns multiple times * update error message when no name * :zap: Update name missing toast message * :zap: Update workflow already exists message * disable saving for executions * fix bug causing modal to close * make tags in workflow open clickable * increase workflow limit to 3 * remove success notifications * update header spacing * escape tag names * update tag and table colors * remove tags from export * build * clean up push connection dependencies * address ben's comments * revert tags optional interface * address comments * update duplicate message * build * fix eol * add one more eol * :zap: Update comment * add hover style for workflow open, fix up font weight Co-authored-by: Mutasem <mutdmour@gmail.com> Co-authored-by: Iván Ovejero <ivov.src@gmail.com> Co-authored-by: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com>
2021-05-29 11:31:21 -07:00
const executionId = typeof executionResult.id === "object" ? executionResult.id!.toString() : executionResult.id + "";
2019-06-23 03:35:23 -07:00
this.activeExecutions[executionId] = {
executionData,
process,
startedAt: new Date(),
2019-06-23 03:35:23 -07:00
postExecutePromises: [],
};
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
return executionId;
2019-06-23 03:35:23 -07:00
}
/**
* Attaches an execution
*
* @param {string} executionId
* @param {PCancelable<IRun>} workflowExecution
* @memberof ActiveExecutions
*/
attachWorkflowExecution(executionId: string, workflowExecution: PCancelable<IRun>) {
if (this.activeExecutions[executionId] === undefined) {
throw new Error(`No active execution with id "${executionId}" got found to attach to workflowExecution to!`);
}
this.activeExecutions[executionId].workflowExecution = workflowExecution;
}
2019-06-23 03:35:23 -07:00
/**
* Remove an active execution
*
* @param {string} executionId
* @param {IRun} fullRunData
* @returns {void}
* @memberof ActiveExecutions
*/
remove(executionId: string, fullRunData?: IRun): void {
2019-06-23 03:35:23 -07:00
if (this.activeExecutions[executionId] === undefined) {
return;
}
// Resolve all the waiting promises
for (const promise of this.activeExecutions[executionId].postExecutePromises) {
promise.resolve(fullRunData);
}
// Remove from the list of active executions
delete this.activeExecutions[executionId];
}
/**
* Forces an execution to stop
*
* @param {string} executionId The id of the execution to stop
* @param {string} timeout String 'timeout' given if stop due to timeout
2019-06-23 03:35:23 -07:00
* @returns {(Promise<IRun | undefined>)}
* @memberof ActiveExecutions
*/
async stopExecution(executionId: string, timeout?: string): Promise<IRun | undefined> {
2019-06-23 03:35:23 -07:00
if (this.activeExecutions[executionId] === undefined) {
// There is no execution running with that id
return;
}
// In case something goes wrong make sure that promise gets first
// returned that it gets then also resolved correctly.
if (this.activeExecutions[executionId].process !== undefined) {
// Workflow is running in subprocess
if (this.activeExecutions[executionId].process!.connected) {
setTimeout(() => {
// execute on next event loop tick;
this.activeExecutions[executionId].process!.send({
type: timeout ? timeout : 'stopExecution',
});
2020-07-29 05:19:35 -07:00
}, 1);
}
} else {
// Workflow is running in current process
this.activeExecutions[executionId].workflowExecution!.cancel();
}
2019-06-23 03:35:23 -07:00
return this.getPostExecutePromise(executionId);
}
/**
* Returns a promise which will resolve with the data of the execution
* with the given id
*
* @param {string} executionId The id of the execution to wait for
* @returns {Promise<IRun>}
* @memberof ActiveExecutions
*/
async getPostExecutePromise(executionId: string): Promise<IRun | undefined> {
2019-06-23 03:35:23 -07:00
// Create the promise which will be resolved when the execution finished
const waitPromise = await createDeferredPromise<IRun | undefined>();
2019-06-23 03:35:23 -07:00
if (this.activeExecutions[executionId] === undefined) {
throw new Error(`There is no active execution with id "${executionId}".`);
}
this.activeExecutions[executionId].postExecutePromises.push(waitPromise);
return waitPromise.promise();
}
/**
* Returns all the currently active executions
*
* @returns {IExecutionsCurrentSummary[]}
* @memberof ActiveExecutions
*/
getActiveExecutions(): IExecutionsCurrentSummary[] {
const returnData: IExecutionsCurrentSummary[] = [];
let data;
2019-06-23 03:35:23 -07:00
for (const id of Object.keys(this.activeExecutions)) {
data = this.activeExecutions[id];
2019-06-23 03:35:23 -07:00
returnData.push(
{
id,
2019-08-08 22:37:10 -07:00
retryOf: data.executionData.retryOf as string | undefined,
startedAt: data.startedAt,
mode: data.executionData.executionMode,
workflowId: data.executionData.workflowData.id! as string,
2019-06-23 03:35:23 -07:00
}
);
}
return returnData;
}
}
let activeExecutionsInstance: ActiveExecutions | undefined;
export function getInstance(): ActiveExecutions {
if (activeExecutionsInstance === undefined) {
activeExecutionsInstance = new ActiveExecutions();
}
return activeExecutionsInstance;
}