n8n/packages/cli/src/Server.ts

1998 lines
71 KiB
TypeScript
Raw Normal View History

2019-06-23 03:35:23 -07:00
import * as express from 'express';
import {
readFileSync,
} from 'fs';
import {
dirname as pathDirname,
join as pathJoin,
resolve as pathResolve,
} from 'path';
import {
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
getConnection,
getConnectionManager,
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
In,
} from 'typeorm';
2019-06-23 03:35:23 -07:00
import * as bodyParser from 'body-parser';
require('body-parser-xml')(bodyParser);
2019-06-23 03:35:23 -07:00
import * as history from 'connect-history-api-fallback';
import * as _ from 'lodash';
import * as clientOAuth2 from 'client-oauth2';
import * as clientOAuth1 from 'oauth-1.0a';
import { RequestOptions } from 'oauth-1.0a';
import * as csrf from 'csrf';
2020-07-14 04:59:37 -07:00
import * as requestPromise from 'request-promise-native';
import { createHmac } from 'crypto';
import { compare } from 'bcryptjs';
2019-06-23 03:35:23 -07:00
import {
ActiveExecutions,
2019-06-23 03:35:23 -07:00
ActiveWorkflowRunner,
CredentialsHelper,
2020-10-22 06:46:03 -07:00
CredentialsOverwrites,
CredentialTypes,
Db,
ExternalHooks,
2020-10-22 06:46:03 -07:00
GenericHelpers,
IActivationError,
2019-06-23 03:35:23 -07:00
ICredentialsDb,
ICredentialsDecryptedDb,
ICredentialsDecryptedResponse,
2020-10-22 06:46:03 -07:00
ICredentialsOverwrite,
2019-06-23 03:35:23 -07:00
ICredentialsResponse,
2020-10-22 06:46:03 -07:00
ICustomRequest,
2019-06-23 03:35:23 -07:00
IExecutionDeleteFilter,
IExecutionFlatted,
IExecutionFlattedDb,
IExecutionFlattedResponse,
IExecutionPushResponse,
2020-10-26 13:26:52 -07:00
IExecutionResponse,
2019-06-23 03:35:23 -07:00
IExecutionsListResponse,
IExecutionsStopData,
IExecutionsSummary,
2020-06-10 04:00:28 -07:00
IExternalHooksClass,
2019-06-23 03:35:23 -07:00
IN8nUISettings,
IPackageVersions,
2019-06-23 03:35:23 -07:00
IWorkflowBase,
IWorkflowExecutionDataProcess,
2020-10-22 06:46:03 -07:00
IWorkflowResponse,
IWorkflowShortResponse,
LoadNodesAndCredentials,
2019-06-23 03:35:23 -07:00
NodeTypes,
Push,
ResponseHelper,
TestWebhooks,
WebhookHelpers,
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
WebhookServer,
2020-10-22 06:46:03 -07:00
WorkflowCredentials,
2019-06-23 03:35:23 -07:00
WorkflowExecuteAdditionalData,
WorkflowRunner,
2019-06-23 03:35:23 -07:00
} from './';
import {
Credentials,
LoadNodeParameterOptions,
UserSettings,
} from 'n8n-core';
import {
ICredentialsEncrypted,
2019-06-23 03:35:23 -07:00
ICredentialType,
IDataObject,
INodeCredentials,
INodeParameters,
2019-06-23 03:35:23 -07:00
INodePropertyOptions,
2020-10-22 06:46:03 -07:00
INodeTypeDescription,
2019-06-23 03:35:23 -07:00
IRunData,
IWorkflowCredentials,
2019-06-23 03:35:23 -07:00
Workflow,
2021-01-29 00:31:40 -08:00
WorkflowExecuteMode,
2019-06-23 03:35:23 -07:00
} from 'n8n-workflow';
import {
FindManyOptions,
FindOneOptions,
2019-06-23 03:35:23 -07:00
LessThan,
LessThanOrEqual,
MoreThanOrEqual,
Not,
2019-06-23 03:35:23 -07:00
} from 'typeorm';
import * as basicAuth from 'basic-auth';
import * as compression from 'compression';
import * as config from '../config';
import * as jwt from 'jsonwebtoken';
import * as jwks from 'jwks-rsa';
2019-06-23 03:35:23 -07:00
// @ts-ignore
import * as timezones from 'google-timezones-json';
import * as parseUrl from 'parseurl';
import * as querystring from 'querystring';
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
import * as Queue from '../src/Queue';
import { OptionsWithUrl } from 'request-promise-native';
2019-06-23 03:35:23 -07:00
class App {
app: express.Application;
activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner;
testWebhooks: TestWebhooks.TestWebhooks;
endpointWebhook: string;
endpointWebhookTest: string;
endpointPresetCredentials: string;
2020-06-10 04:00:28 -07:00
externalHooks: IExternalHooksClass;
saveDataErrorExecution: string;
saveDataSuccessExecution: string;
saveManualExecutions: boolean;
executionTimeout: number;
maxExecutionTimeout: number;
2019-06-23 03:35:23 -07:00
timezone: string;
activeExecutionsInstance: ActiveExecutions.ActiveExecutions;
push: Push.Push;
versions: IPackageVersions | undefined;
2020-06-23 07:12:53 -07:00
restEndpoint: string;
frontendSettings: IN8nUISettings;
protocol: string;
2020-07-14 04:59:37 -07:00
sslKey: string;
sslCert: string;
presetCredentialsLoaded: boolean;
2019-06-23 03:35:23 -07:00
constructor() {
this.app = express();
this.endpointWebhook = config.get('endpoints.webhook') as string;
this.endpointWebhookTest = config.get('endpoints.webhookTest') as string;
this.saveDataErrorExecution = config.get('executions.saveDataOnError') as string;
this.saveDataSuccessExecution = config.get('executions.saveDataOnSuccess') as string;
this.saveManualExecutions = config.get('executions.saveDataManualExecutions') as boolean;
this.executionTimeout = config.get('executions.timeout') as number;
this.maxExecutionTimeout = config.get('executions.maxTimeout') as number;
this.timezone = config.get('generic.timezone') as string;
2020-06-23 07:12:53 -07:00
this.restEndpoint = config.get('endpoints.rest') as string;
2019-06-23 03:35:23 -07:00
this.activeWorkflowRunner = ActiveWorkflowRunner.getInstance();
this.testWebhooks = TestWebhooks.getInstance();
this.push = Push.getInstance();
this.activeExecutionsInstance = ActiveExecutions.getInstance();
this.protocol = config.get('protocol');
2020-07-14 04:59:37 -07:00
this.sslKey = config.get('ssl_key');
this.sslCert = config.get('ssl_cert');
this.externalHooks = ExternalHooks();
this.presetCredentialsLoaded = false;
this.endpointPresetCredentials = config.get('credentials.overwrite.endpoint') as string;
const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
this.frontendSettings = {
endpointWebhook: this.endpointWebhook,
endpointWebhookTest: this.endpointWebhookTest,
saveDataErrorExecution: this.saveDataErrorExecution,
saveDataSuccessExecution: this.saveDataSuccessExecution,
saveManualExecutions: this.saveManualExecutions,
executionTimeout: this.executionTimeout,
maxExecutionTimeout: this.maxExecutionTimeout,
timezone: this.timezone,
urlBaseWebhook,
versionCli: '',
oauthCallbackUrls: {
'oauth1': urlBaseWebhook + `${this.restEndpoint}/oauth1-credential/callback`,
'oauth2': urlBaseWebhook + `${this.restEndpoint}/oauth2-credential/callback`,
2020-10-22 06:46:03 -07:00
},
};
2019-06-23 03:35:23 -07:00
}
/**
* Returns the current epoch time
*
* @returns {number}
* @memberof App
*/
getCurrentDate(): Date {
return new Date();
2019-06-23 03:35:23 -07:00
}
async config(): Promise<void> {
2019-06-23 03:35:23 -07:00
this.versions = await GenericHelpers.getVersions();
this.frontendSettings.versionCli = this.versions.cli;
await this.externalHooks.run('frontend.settings', [this.frontendSettings]);
const excludeEndpoints = config.get('security.excludeEndpoints') as string;
const ignoredEndpoints = ['healthz', this.endpointWebhook, this.endpointWebhookTest, this.endpointPresetCredentials];
ignoredEndpoints.push.apply(ignoredEndpoints, excludeEndpoints.split(':'));
const authIgnoreRegex = new RegExp(`^\/(${_(ignoredEndpoints).compact().join('|')})\/?.*$`);
// Check for basic auth credentials if activated
const basicAuthActive = config.get('security.basicAuth.active') as boolean;
if (basicAuthActive === true) {
const basicAuthUser = await GenericHelpers.getConfigValue('security.basicAuth.user') as string;
if (basicAuthUser === '') {
throw new Error('Basic auth is activated but no user got defined. Please set one!');
}
const basicAuthPassword = await GenericHelpers.getConfigValue('security.basicAuth.password') as string;
if (basicAuthPassword === '') {
throw new Error('Basic auth is activated but no password got defined. Please set one!');
}
const basicAuthHashEnabled = await GenericHelpers.getConfigValue('security.basicAuth.hash') as boolean;
let validPassword: null | string = null;
this.app.use(async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (req.url.match(authIgnoreRegex)) {
return next();
}
const realm = 'n8n - Editor UI';
const basicAuthData = basicAuth(req);
if (basicAuthData === undefined) {
// Authorization data is missing
return ResponseHelper.basicAuthAuthorizationError(res, realm, 'Authorization is required!');
}
if (basicAuthData.name === basicAuthUser) {
if (basicAuthHashEnabled === true) {
if (validPassword === null && await compare(basicAuthData.pass, basicAuthPassword)) {
// Password is valid so save for future requests
validPassword = basicAuthData.pass;
}
if (validPassword === basicAuthData.pass && validPassword !== null) {
// Provided hash is correct
return next();
}
} else {
if (basicAuthData.pass === basicAuthPassword) {
// Provided password is correct
return next();
}
}
}
// Provided authentication data is wrong
return ResponseHelper.basicAuthAuthorizationError(res, realm, 'Authorization data is wrong!');
});
}
// Check for and validate JWT if configured
2020-07-14 04:59:37 -07:00
const jwtAuthActive = config.get('security.jwtAuth.active') as boolean;
if (jwtAuthActive === true) {
const jwtAuthHeader = await GenericHelpers.getConfigValue('security.jwtAuth.jwtHeader') as string;
if (jwtAuthHeader === '') {
throw new Error('JWT auth is activated but no request header was defined. Please set one!');
}
const jwksUri = await GenericHelpers.getConfigValue('security.jwtAuth.jwksUri') as string;
if (jwksUri === '') {
throw new Error('JWT auth is activated but no JWK Set URI was defined. Please set one!');
}
const jwtHeaderValuePrefix = await GenericHelpers.getConfigValue('security.jwtAuth.jwtHeaderValuePrefix') as string;
const jwtIssuer = await GenericHelpers.getConfigValue('security.jwtAuth.jwtIssuer') as string;
const jwtNamespace = await GenericHelpers.getConfigValue('security.jwtAuth.jwtNamespace') as string;
const jwtAllowedTenantKey = await GenericHelpers.getConfigValue('security.jwtAuth.jwtAllowedTenantKey') as string;
const jwtAllowedTenant = await GenericHelpers.getConfigValue('security.jwtAuth.jwtAllowedTenant') as string;
function isTenantAllowed(decodedToken: object): boolean {
if (jwtNamespace === '' || jwtAllowedTenantKey === '' || jwtAllowedTenant === '') return true;
else {
for (const [k, v] of Object.entries(decodedToken)) {
if (k === jwtNamespace) {
for (const [kn, kv] of Object.entries(v)) {
if (kn === jwtAllowedTenantKey && kv === jwtAllowedTenant) {
return true;
}
}
}
}
}
return false;
}
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
if (req.url.match(authIgnoreRegex)) {
return next();
}
let token = req.header(jwtAuthHeader) as string;
if (token === undefined || token === '') {
return ResponseHelper.jwtAuthAuthorizationError(res, "Missing token");
}
2020-09-01 00:22:41 -07:00
if (jwtHeaderValuePrefix !== '' && token.startsWith(jwtHeaderValuePrefix)) {
token = token.replace(jwtHeaderValuePrefix + ' ', '').trimLeft();
}
const jwkClient = jwks({ cache: true, jwksUri });
function getKey(header: any, callback: Function) { // tslint:disable-line:no-any
jwkClient.getSigningKey(header.kid, (err: Error, key: any) => { // tslint:disable-line:no-any
if (err) throw ResponseHelper.jwtAuthAuthorizationError(res, err.message);
const signingKey = key.publicKey || key.rsaPublicKey;
callback(null, signingKey);
});
}
const jwtVerifyOptions: jwt.VerifyOptions = {
issuer: jwtIssuer !== '' ? jwtIssuer : undefined,
2020-10-22 06:46:03 -07:00
ignoreExpiration: false,
};
jwt.verify(token, getKey, jwtVerifyOptions, (err: jwt.VerifyErrors, decoded: object) => {
if (err) ResponseHelper.jwtAuthAuthorizationError(res, 'Invalid token');
else if (!isTenantAllowed(decoded)) ResponseHelper.jwtAuthAuthorizationError(res, 'Tenant not allowed');
else next();
});
});
}
2019-06-23 03:35:23 -07:00
// Get push connections
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
2020-06-23 07:12:53 -07:00
if (req.url.indexOf(`/${this.restEndpoint}/push`) === 0) {
2019-06-23 03:35:23 -07:00
// TODO: Later also has to add some kind of authentication token
if (req.query.sessionId === undefined) {
next(new Error('The query parameter "sessionId" is missing!'));
return;
}
this.push.add(req.query.sessionId as string, req, res);
2019-06-23 03:35:23 -07:00
return;
}
next();
});
2019-08-06 11:44:36 -07:00
// Compress the response data
this.app.use(compression());
2019-06-23 03:35:23 -07:00
// Make sure that each request has the "parsedUrl" parameter
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
(req as ICustomRequest).parsedUrl = parseUrl(req);
// @ts-ignore
req.rawBody = Buffer.from('', 'base64');
2019-06-23 03:35:23 -07:00
next();
});
// Support application/json type post data
2020-02-09 18:27:06 -08:00
this.app.use(bodyParser.json({
limit: '16mb', verify: (req, res, buf) => {
// @ts-ignore
req.rawBody = buf;
2020-10-22 06:46:03 -07:00
},
2020-02-09 18:27:06 -08:00
}));
2019-06-23 03:35:23 -07:00
// Support application/xml type post data
// @ts-ignore
this.app.use(bodyParser.xml({
limit: '16mb', xmlParseOptions: {
normalize: true, // Trim whitespace inside text nodes
normalizeTags: true, // Transform tags to lowercase
explicitArray: false, // Only put properties in array if length > 1
2020-10-22 06:46:03 -07:00
},
}));
2020-02-09 18:27:06 -08:00
this.app.use(bodyParser.text({
limit: '16mb', verify: (req, res, buf) => {
// @ts-ignore
req.rawBody = buf;
2020-10-22 06:46:03 -07:00
},
2020-02-09 18:27:06 -08:00
}));
2019-06-23 03:35:23 -07:00
// Make sure that Vue history mode works properly
this.app.use(history({
rewrites: [
{
2020-06-23 07:12:53 -07:00
from: new RegExp(`^\/(${this.restEndpoint}|healthz|css|js|${this.endpointWebhook}|${this.endpointWebhookTest})\/?.*$`),
2019-06-23 03:35:23 -07:00
to: (context) => {
return context.parsedUrl!.pathname!.toString();
2020-10-22 06:46:03 -07:00
},
},
],
2019-06-23 03:35:23 -07:00
}));
//support application/x-www-form-urlencoded post data
:sparkles: Add QuickBooks node (#1365) * Add OAuth2 credentials * Adjust credentials params * Add node to listing * Add initial node scaffolding * Remove unused credentials params * Add customer search with select statement * Add pagination to customer search * Add customer creation functionality * Add customer update functionality * Small formatting fix * Adjust property name casing for consistency * Adjust customer operations for consistency * Handle large QuickBooks listings * Add initial estimate resource description * Add estimate resource retrieval operations * Refactor customer billing address * Simplify customer billing address * Fix casing for customer additional fields * Adjust types to accommodate loadOptions functions * Add loadOptions for customers to estimate * Sort customer update fields alphabetically * Refactor estimate line into standalone file * Add stub for PDF retrieval operation * Add invoice resource description and execute branches * Implement estimate PDF download functionality * Place descriptions in their own dir * Add get and getAll for invoices * Add send functionality to invoices * Refactor handling of binary data * Add invoice voiding functionality * Add invoice deletion functionality * Refactor resources into subdirs and add interfaces * Add get and getAll for bill * Add payment description * Add get and getAll for payment * Make variables in endpoints consistent * Refactor interfaces for consistency * Add interface for item resource * Fill in fields for all resources * Minor fixes in defaults and descriptions * Refactor loader * Add all resources to execute function * Fix line property duplication * Add get and getAll for vendor * Optimize description imports * Add creation for customer and bill * Add update operation for bill * Refactor create and update for customer * Implement employee create and update * Implement create and update for estimate * Make address params more consistent * Add create and update to payment * Add item operations * Add create and delete operations for invoice * Refactor binary data handler * Refactor generic functions * Add create and update operations for vendor * Fix build * Fix total amount in bill:update * Fix balance in bill:update * Remove currency from bill:update * Implement reference retrieval in bill:update * Fix failing params in customer:update * Fix param in employee:update * Implement reference retrieval in estimate:update * Fix failing params in estimate:update * Fix failing params in invoice:update * Fix failing param in vendor:update * Implement reference retrieval in payment:update * Remove unused interfaces * Rename line property function * Remove hared directory * Refactor reference and sync token retrieval * Fix line structure in bill:create * Fix line structure in estimate:create * Flatten responses * Refactor line processing * Remove unused interfaces * Add endpoint documentation * Fix payment:void content type * Fix default for bill line item * Hide auth URI query parameters * Hide auth header parameter * Add switch for credentials environment * Adjust OAuth2 callback to accommodate realmId * Retrieve realmId from OAuth2 flow * :zap: Improvements * Reposition dividers * Add IDs to display names of reference fields * Add estimate:delete and bill:delete * Load items in lines for bill, estimate and invoice * Add filename for binary property in PDF download * :zap: Improvements * Adjust field description * Implement estimate:send * Adjust field description * Adjust custom field descriptions * Add missing period to description * :zap: Minor improvements on QuickBooks-Node * Add descriptions for bill * Add descriptions for customer * Add descriptions for employee * Add descriptions for estimate * Add descriptions for invoice * Add descriptions for payment * Add descriptions for vendor Co-authored-by: ricardo <ricardoespinoza105@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-13 08:27:08 -08:00
this.app.use(bodyParser.urlencoded({
extended: false,
verify: (req, res, buf) => {
// @ts-ignore
req.rawBody = buf;
2020-10-22 06:46:03 -07:00
},
}));
2019-06-23 03:35:23 -07:00
if (process.env['NODE_ENV'] !== 'production') {
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
// Allow access also from frontend when developing
res.header('Access-Control-Allow-Origin', 'http://localhost:8080');
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE');
res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, sessionid');
next();
});
}
2019-06-23 03:35:23 -07:00
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
if (Db.collections.Workflow === null) {
2019-08-28 08:16:09 -07:00
const error = new ResponseHelper.ResponseError('Database is not ready!', undefined, 503);
2019-06-23 03:35:23 -07:00
return ResponseHelper.sendErrorResponse(res, error);
}
next();
});
// ----------------------------------------
// Healthcheck
// ----------------------------------------
// Does very basic health check
this.app.get('/healthz', async (req: express.Request, res: express.Response) => {
const connectionManager = getConnectionManager();
if (connectionManager.connections.length === 0) {
const error = new ResponseHelper.ResponseError('No Database connection found!', undefined, 503);
return ResponseHelper.sendErrorResponse(res, error);
}
if (connectionManager.connections[0].isConnected === false) {
// Connection is not active
const error = new ResponseHelper.ResponseError('Database connection not active!', undefined, 503);
return ResponseHelper.sendErrorResponse(res, error);
}
// Everything fine
const responseData = {
status: 'ok',
};
ResponseHelper.sendSuccessResponse(res, responseData, true, 200);
});
2019-06-23 03:35:23 -07:00
// ----------------------------------------
// Workflow
// ----------------------------------------
// Creates a new workflow
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/workflows`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse> => {
2019-06-23 03:35:23 -07:00
const newWorkflowData = req.body as IWorkflowBase;
2019-06-23 03:35:23 -07:00
newWorkflowData.name = newWorkflowData.name.trim();
2019-06-23 03:35:23 -07:00
newWorkflowData.createdAt = this.getCurrentDate();
newWorkflowData.updatedAt = this.getCurrentDate();
newWorkflowData.id = undefined;
await this.externalHooks.run('workflow.create', [newWorkflowData]);
2019-06-23 03:35:23 -07:00
// Save the workflow in DB
const result = await Db.collections.Workflow!.save(newWorkflowData);
// Convert to response format in which the id is a string
(result as IWorkflowBase as IWorkflowResponse).id = result.id.toString();
return result as IWorkflowBase as IWorkflowResponse;
}));
// Reads and returns workflow data from an URL
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/workflows/from-url`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse> => {
2019-06-23 03:35:23 -07:00
if (req.query.url === undefined) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`The parameter "url" is missing!`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
if (!(req.query.url as string).match(/^http[s]?:\/\/.*\.json$/i)) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`The parameter "url" is not valid! It does not seem to be a URL pointing to a n8n workflow JSON file.`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
const data = await requestPromise.get(req.query.url as string);
2019-06-23 03:35:23 -07:00
let workflowData: IWorkflowResponse | undefined;
try {
workflowData = JSON.parse(data);
} catch (error) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`The URL does not point to valid JSON file!`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
// Do a very basic check if it is really a n8n-workflow-json
if (workflowData === undefined || workflowData.nodes === undefined || !Array.isArray(workflowData.nodes) ||
workflowData.connections === undefined || typeof workflowData.connections !== 'object' ||
Array.isArray(workflowData.connections)) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`The data in the file does not seem to be a n8n workflow JSON file!`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
return workflowData;
}));
// Returns workflows
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/workflows`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowShortResponse[]> => {
2019-06-23 03:35:23 -07:00
const findQuery = {} as FindManyOptions;
if (req.query.filter) {
findQuery.where = JSON.parse(req.query.filter as string);
2019-06-23 03:35:23 -07:00
}
// Return only the fields we need
findQuery.select = ['id', 'name', 'active', 'createdAt', 'updatedAt'];
const results = await Db.collections.Workflow!.find(findQuery);
for (const entry of results) {
(entry as unknown as IWorkflowShortResponse).id = entry.id.toString();
}
return results as unknown as IWorkflowShortResponse[];
}));
// Returns a specific workflow
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/workflows/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse | undefined> => {
2019-06-23 03:35:23 -07:00
const result = await Db.collections.Workflow!.findOne(req.params.id);
if (result === undefined) {
return undefined;
}
// Convert to response format in which the id is a string
(result as IWorkflowBase as IWorkflowResponse).id = result.id.toString();
return result as IWorkflowBase as IWorkflowResponse;
}));
// Updates an existing workflow
2020-06-23 07:12:53 -07:00
this.app.patch(`/${this.restEndpoint}/workflows/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse> => {
2019-06-23 03:35:23 -07:00
const newWorkflowData = req.body as IWorkflowBase;
2019-06-23 03:35:23 -07:00
const id = req.params.id;
await this.externalHooks.run('workflow.update', [newWorkflowData]);
2020-05-27 16:32:49 -07:00
const isActive = await this.activeWorkflowRunner.isActive(id);
if (isActive) {
2019-06-23 03:35:23 -07:00
// When workflow gets saved always remove it as the triggers could have been
// changed and so the changes would not take effect
await this.activeWorkflowRunner.remove(id);
}
if (newWorkflowData.settings) {
if (newWorkflowData.settings.timezone === 'DEFAULT') {
// Do not save the default timezone
delete newWorkflowData.settings.timezone;
}
if (newWorkflowData.settings.saveDataErrorExecution === 'DEFAULT') {
// Do not save when default got set
delete newWorkflowData.settings.saveDataErrorExecution;
}
if (newWorkflowData.settings.saveDataSuccessExecution === 'DEFAULT') {
// Do not save when default got set
delete newWorkflowData.settings.saveDataSuccessExecution;
}
if (newWorkflowData.settings.saveManualExecutions === 'DEFAULT') {
2019-06-23 03:35:23 -07:00
// Do not save when default got set
delete newWorkflowData.settings.saveManualExecutions;
2019-06-23 03:35:23 -07:00
}
2020-07-29 05:19:35 -07:00
if (parseInt(newWorkflowData.settings.executionTimeout as string, 10) === this.executionTimeout) {
// Do not save when default got set
2020-07-29 05:19:35 -07:00
delete newWorkflowData.settings.executionTimeout;
}
2019-06-23 03:35:23 -07:00
}
newWorkflowData.updatedAt = this.getCurrentDate();
await Db.collections.Workflow!.update(id, newWorkflowData);
await this.externalHooks.run('workflow.afterUpdate', [newWorkflowData]);
2019-06-23 03:35:23 -07:00
// We sadly get nothing back from "update". Neither if it updated a record
// nor the new value. So query now the hopefully updated entry.
2019-08-28 08:16:09 -07:00
const responseData = await Db.collections.Workflow!.findOne(id);
2019-06-23 03:35:23 -07:00
2019-08-28 08:16:09 -07:00
if (responseData === undefined) {
throw new ResponseHelper.ResponseError(`Workflow with id "${id}" could not be found to be updated.`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
2019-08-28 08:16:09 -07:00
if (responseData.active === true) {
2019-06-23 03:35:23 -07:00
// When the workflow is supposed to be active add it again
try {
await this.externalHooks.run('workflow.activate', [responseData]);
2019-06-23 03:35:23 -07:00
await this.activeWorkflowRunner.add(id);
} catch (error) {
// If workflow could not be activated set it again to inactive
newWorkflowData.active = false;
await Db.collections.Workflow!.update(id, newWorkflowData);
// Also set it in the returned data
2019-08-28 08:16:09 -07:00
responseData.active = false;
2019-06-23 03:35:23 -07:00
// Now return the original error for UI to display
throw error;
}
}
// Convert to response format in which the id is a string
2019-08-28 08:16:09 -07:00
(responseData as IWorkflowBase as IWorkflowResponse).id = responseData.id.toString();
return responseData as IWorkflowBase as IWorkflowResponse;
2019-06-23 03:35:23 -07:00
}));
// Deletes a specific workflow
2020-06-23 07:12:53 -07:00
this.app.delete(`/${this.restEndpoint}/workflows/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
2019-06-23 03:35:23 -07:00
const id = req.params.id;
await this.externalHooks.run('workflow.delete', [id]);
2020-05-27 16:32:49 -07:00
const isActive = await this.activeWorkflowRunner.isActive(id);
if (isActive) {
2019-06-23 03:35:23 -07:00
// Before deleting a workflow deactivate it
await this.activeWorkflowRunner.remove(id);
}
await Db.collections.Workflow!.delete(id);
2020-10-14 08:47:43 -07:00
await this.externalHooks.run('workflow.afterDelete', [id]);
2019-06-23 03:35:23 -07:00
return true;
}));
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/workflows/run`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IExecutionPushResponse> => {
2019-06-23 03:35:23 -07:00
const workflowData = req.body.workflowData;
const runData: IRunData | undefined = req.body.runData;
const startNodes: string[] | undefined = req.body.startNodes;
const destinationNode: string | undefined = req.body.destinationNode;
const executionMode = 'manual';
const sessionId = GenericHelpers.getSessionId(req);
// If webhooks nodes exist and are active we have to wait for till we receive a call
if (runData === undefined || startNodes === undefined || startNodes.length === 0 || destinationNode === undefined) {
const credentials = await WorkflowCredentials(workflowData.nodes);
2019-12-19 14:07:55 -08:00
const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials);
const nodeTypes = NodeTypes();
const workflowInstance = new Workflow({ id: workflowData.id, name: workflowData.name, nodes: workflowData.nodes, connections: workflowData.connections, active: false, nodeTypes, staticData: undefined, settings: workflowData.settings });
const needsWebhook = await this.testWebhooks.needsWebhookData(workflowData, workflowInstance, additionalData, executionMode, sessionId, destinationNode);
if (needsWebhook === true) {
return {
waitingForWebhook: true,
};
}
}
2019-06-23 03:35:23 -07:00
// For manual testing always set to not active
workflowData.active = false;
2019-06-23 03:35:23 -07:00
const credentials = await WorkflowCredentials(workflowData.nodes);
2019-06-23 03:35:23 -07:00
// Start the workflow
const data: IWorkflowExecutionDataProcess = {
credentials,
destinationNode,
executionMode,
runData,
sessionId,
startNodes,
workflowData,
};
const workflowRunner = new WorkflowRunner();
const executionId = await workflowRunner.run(data);
2019-06-23 03:35:23 -07:00
return {
executionId,
};
}));
// Returns parameter values which normally get loaded from an external API or
// get generated dynamically
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/node-parameter-options`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<INodePropertyOptions[]> => {
const nodeType = req.query.nodeType as string;
2019-06-23 03:35:23 -07:00
let credentials: INodeCredentials | undefined = undefined;
const currentNodeParameters = JSON.parse('' + req.query.currentNodeParameters) as INodeParameters;
2019-06-23 03:35:23 -07:00
if (req.query.credentials !== undefined) {
credentials = JSON.parse(req.query.credentials as string);
2019-06-23 03:35:23 -07:00
}
const methodName = req.query.methodName as string;
2019-06-23 03:35:23 -07:00
const nodeTypes = NodeTypes();
const loadDataInstance = new LoadNodeParameterOptions(nodeType, nodeTypes, JSON.parse('' + req.query.currentNodeParameters), credentials!);
2019-06-23 03:35:23 -07:00
const workflowData = loadDataInstance.getWorkflowData() as IWorkflowBase;
const workflowCredentials = await WorkflowCredentials(workflowData.nodes);
2019-12-19 14:07:55 -08:00
const additionalData = await WorkflowExecuteAdditionalData.getBase(workflowCredentials, currentNodeParameters);
2019-06-23 03:35:23 -07:00
return loadDataInstance.getOptions(methodName, additionalData);
}));
// Returns all the node-types
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/node-types`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<INodeTypeDescription[]> => {
2019-06-23 03:35:23 -07:00
const returnData: INodeTypeDescription[] = [];
const nodeTypes = NodeTypes();
const allNodes = nodeTypes.getAll();
allNodes.forEach((nodeData) => {
// Make a copy of the object. If we don't do this, then when
// The method below is called the properties are removed for good
// This happens because nodes are returned as reference.
:sparkles: Add QuickBooks node (#1365) * Add OAuth2 credentials * Adjust credentials params * Add node to listing * Add initial node scaffolding * Remove unused credentials params * Add customer search with select statement * Add pagination to customer search * Add customer creation functionality * Add customer update functionality * Small formatting fix * Adjust property name casing for consistency * Adjust customer operations for consistency * Handle large QuickBooks listings * Add initial estimate resource description * Add estimate resource retrieval operations * Refactor customer billing address * Simplify customer billing address * Fix casing for customer additional fields * Adjust types to accommodate loadOptions functions * Add loadOptions for customers to estimate * Sort customer update fields alphabetically * Refactor estimate line into standalone file * Add stub for PDF retrieval operation * Add invoice resource description and execute branches * Implement estimate PDF download functionality * Place descriptions in their own dir * Add get and getAll for invoices * Add send functionality to invoices * Refactor handling of binary data * Add invoice voiding functionality * Add invoice deletion functionality * Refactor resources into subdirs and add interfaces * Add get and getAll for bill * Add payment description * Add get and getAll for payment * Make variables in endpoints consistent * Refactor interfaces for consistency * Add interface for item resource * Fill in fields for all resources * Minor fixes in defaults and descriptions * Refactor loader * Add all resources to execute function * Fix line property duplication * Add get and getAll for vendor * Optimize description imports * Add creation for customer and bill * Add update operation for bill * Refactor create and update for customer * Implement employee create and update * Implement create and update for estimate * Make address params more consistent * Add create and update to payment * Add item operations * Add create and delete operations for invoice * Refactor binary data handler * Refactor generic functions * Add create and update operations for vendor * Fix build * Fix total amount in bill:update * Fix balance in bill:update * Remove currency from bill:update * Implement reference retrieval in bill:update * Fix failing params in customer:update * Fix param in employee:update * Implement reference retrieval in estimate:update * Fix failing params in estimate:update * Fix failing params in invoice:update * Fix failing param in vendor:update * Implement reference retrieval in payment:update * Remove unused interfaces * Rename line property function * Remove hared directory * Refactor reference and sync token retrieval * Fix line structure in bill:create * Fix line structure in estimate:create * Flatten responses * Refactor line processing * Remove unused interfaces * Add endpoint documentation * Fix payment:void content type * Fix default for bill line item * Hide auth URI query parameters * Hide auth header parameter * Add switch for credentials environment * Adjust OAuth2 callback to accommodate realmId * Retrieve realmId from OAuth2 flow * :zap: Improvements * Reposition dividers * Add IDs to display names of reference fields * Add estimate:delete and bill:delete * Load items in lines for bill, estimate and invoice * Add filename for binary property in PDF download * :zap: Improvements * Adjust field description * Implement estimate:send * Adjust field description * Adjust custom field descriptions * Add missing period to description * :zap: Minor improvements on QuickBooks-Node * Add descriptions for bill * Add descriptions for customer * Add descriptions for employee * Add descriptions for estimate * Add descriptions for invoice * Add descriptions for payment * Add descriptions for vendor Co-authored-by: ricardo <ricardoespinoza105@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-13 08:27:08 -08:00
const nodeInfo: INodeTypeDescription = { ...nodeData.description };
if (req.query.includeProperties !== 'true') {
// @ts-ignore
delete nodeInfo.properties;
}
returnData.push(nodeInfo);
2019-06-23 03:35:23 -07:00
});
return returnData;
}));
// Returns node information baesd on namese
this.app.post(`/${this.restEndpoint}/node-types`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<INodeTypeDescription[]> => {
const nodeNames = _.get(req, 'body.nodeNames', []) as string[];
const nodeTypes = NodeTypes();
return nodeNames.map(name => {
try {
return nodeTypes.getByName(name);
} catch (e) {
return undefined;
}
}).filter(nodeData => !!nodeData).map(nodeData => nodeData!.description);
}));
2019-06-23 03:35:23 -07:00
// ----------------------------------------
// Node-Types
// ----------------------------------------
// Returns the node icon
2020-06-23 07:12:53 -07:00
this.app.get([`/${this.restEndpoint}/node-icon/:nodeType`, `/${this.restEndpoint}/node-icon/:scope/:nodeType`], async (req: express.Request, res: express.Response): Promise<void> => {
const nodeTypeName = `${req.params.scope ? `${req.params.scope}/` : ''}${req.params.nodeType}`;
2019-06-23 03:35:23 -07:00
const nodeTypes = NodeTypes();
const nodeType = nodeTypes.getByName(nodeTypeName);
if (nodeType === undefined) {
res.status(404).send('The nodeType is not known.');
return;
}
if (nodeType.description.icon === undefined) {
res.status(404).send('No icon found for node.');
return;
}
if (!nodeType.description.icon.startsWith('file:')) {
res.status(404).send('Node does not have a file icon.');
return;
}
const filepath = nodeType.description.icon.substr(5);
res.sendFile(filepath);
});
// ----------------------------------------
// Active Workflows
// ----------------------------------------
// Returns the active workflow ids
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/active`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string[]> => {
2020-05-27 16:32:49 -07:00
const activeWorkflows = await this.activeWorkflowRunner.getActiveWorkflows();
return activeWorkflows.map(workflow => workflow.id.toString()) as string[];
2019-06-23 03:35:23 -07:00
}));
// Returns if the workflow with the given id had any activation errors
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/active/error/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IActivationError | undefined> => {
2019-06-23 03:35:23 -07:00
const id = req.params.id;
return this.activeWorkflowRunner.getActivationError(id);
}));
// ----------------------------------------
// Credentials
// ----------------------------------------
// Deletes a specific credential
2020-06-23 07:12:53 -07:00
this.app.delete(`/${this.restEndpoint}/credentials/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
2019-06-23 03:35:23 -07:00
const id = req.params.id;
await this.externalHooks.run('credentials.delete', [id]);
2019-06-23 03:35:23 -07:00
await Db.collections.Credentials!.delete({ id });
return true;
}));
// Creates new credentials
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/credentials`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialsResponse> => {
2019-06-23 03:35:23 -07:00
const incomingData = req.body;
if (!incomingData.name || incomingData.name.length < 3) {
throw new ResponseHelper.ResponseError(`Credentials name must be at least 3 characters long.`, undefined, 400);
}
2019-06-23 03:35:23 -07:00
// Add the added date for node access permissions
for (const nodeAccess of incomingData.nodesAccess) {
nodeAccess.date = this.getCurrentDate();
}
const encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
throw new Error('No encryption key got found to encrypt the credentials!');
}
if (incomingData.name === '') {
throw new Error('Credentials have to have a name set!');
}
// Check if credentials with the same name and type exist already
const findQuery = {
where: {
name: incomingData.name,
type: incomingData.type,
},
} as FindOneOptions;
const checkResult = await Db.collections.Credentials!.findOne(findQuery);
if (checkResult !== undefined) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`Credentials with the same type and name exist already.`, undefined, 400);
}
2019-06-23 03:35:23 -07:00
// Encrypt the data
const credentials = new Credentials(incomingData.name, incomingData.type, incomingData.nodesAccess);
credentials.setData(incomingData.data, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as ICredentialsDb;
await this.externalHooks.run('credentials.create', [newCredentialsData]);
2019-06-23 03:35:23 -07:00
// Add special database related data
newCredentialsData.createdAt = this.getCurrentDate();
newCredentialsData.updatedAt = this.getCurrentDate();
// TODO: also add user automatically depending on who is logged in, if anybody is logged in
// Save the credentials in DB
const result = await Db.collections.Credentials!.save(newCredentialsData);
result.data = incomingData.data;
2019-06-23 03:35:23 -07:00
// Convert to response format in which the id is a string
(result as unknown as ICredentialsResponse).id = result.id.toString();
return result as unknown as ICredentialsResponse;
}));
// Updates existing credentials
2020-06-23 07:12:53 -07:00
this.app.patch(`/${this.restEndpoint}/credentials/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialsResponse> => {
2019-06-23 03:35:23 -07:00
const incomingData = req.body;
const id = req.params.id;
if (incomingData.name === '') {
throw new Error('Credentials have to have a name set!');
}
2019-06-23 03:35:23 -07:00
// Add the date for newly added node access permissions
for (const nodeAccess of incomingData.nodesAccess) {
if (!nodeAccess.date) {
nodeAccess.date = this.getCurrentDate();
}
}
// Check if credentials with the same name and type exist already
const findQuery = {
where: {
id: Not(id),
name: incomingData.name,
type: incomingData.type,
},
} as FindOneOptions;
const checkResult = await Db.collections.Credentials!.findOne(findQuery);
if (checkResult !== undefined) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`Credentials with the same type and name exist already.`, undefined, 400);
}
2019-06-23 03:35:23 -07:00
const encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
throw new Error('No encryption key got found to encrypt the credentials!');
}
// Load the currently saved credentials to be able to persist some of the data if
const result = await Db.collections.Credentials!.findOne(id);
if (result === undefined) {
throw new ResponseHelper.ResponseError(`Credentials with the id "${id}" do not exist.`, undefined, 400);
}
const currentlySavedCredentials = new Credentials(result.name, result.type, result.nodesAccess, result.data);
const decryptedData = currentlySavedCredentials.getData(encryptionKey!);
// Do not overwrite the oauth data else data like the access or refresh token would get lost
// everytime anybody changes anything on the credentials even if it is just the name.
if (decryptedData.oauthTokenData) {
incomingData.data.oauthTokenData = decryptedData.oauthTokenData;
}
2019-06-23 03:35:23 -07:00
// Encrypt the data
const credentials = new Credentials(incomingData.name, incomingData.type, incomingData.nodesAccess);
credentials.setData(incomingData.data, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
// Add special database related data
newCredentialsData.updatedAt = this.getCurrentDate();
await this.externalHooks.run('credentials.update', [newCredentialsData]);
2019-06-23 03:35:23 -07:00
// Update the credentials in DB
await Db.collections.Credentials!.update(id, newCredentialsData);
// We sadly get nothing back from "update". Neither if it updated a record
// nor the new value. So query now the hopefully updated entry.
2019-08-28 08:16:09 -07:00
const responseData = await Db.collections.Credentials!.findOne(id);
2019-06-23 03:35:23 -07:00
2019-08-28 08:16:09 -07:00
if (responseData === undefined) {
throw new ResponseHelper.ResponseError(`Credentials with id "${id}" could not be found to be updated.`, undefined, 400);
2019-06-23 03:35:23 -07:00
}
// Remove the encrypted data as it is not needed in the frontend
2019-08-28 08:16:09 -07:00
responseData.data = '';
2019-06-23 03:35:23 -07:00
// Convert to response format in which the id is a string
2019-08-28 08:16:09 -07:00
(responseData as unknown as ICredentialsResponse).id = responseData.id.toString();
return responseData as unknown as ICredentialsResponse;
2019-06-23 03:35:23 -07:00
}));
// Returns specific credentials
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/credentials/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialsDecryptedResponse | ICredentialsResponse | undefined> => {
2019-06-23 03:35:23 -07:00
const findQuery = {} as FindManyOptions;
// Make sure the variable has an expected value
const includeData = ['true', true].includes(req.query.includeData as string);
2019-06-23 03:35:23 -07:00
if (includeData !== true) {
2019-06-23 03:35:23 -07:00
// Return only the fields we need
findQuery.select = ['id', 'name', 'type', 'nodesAccess', 'createdAt', 'updatedAt'];
}
const result = await Db.collections.Credentials!.findOne(req.params.id);
if (result === undefined) {
return result;
}
let encryptionKey = undefined;
if (includeData === true) {
2019-06-23 03:35:23 -07:00
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
throw new Error('No encryption key got found to decrypt the credentials!');
}
const credentials = new Credentials(result.name, result.type, result.nodesAccess, result.data);
(result as ICredentialsDecryptedDb).data = credentials.getData(encryptionKey!);
}
(result as ICredentialsDecryptedResponse).id = result.id.toString();
return result as ICredentialsDecryptedResponse;
}));
// Returns all the saved credentials
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/credentials`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialsResponse[]> => {
2019-06-23 03:35:23 -07:00
const findQuery = {} as FindManyOptions;
if (req.query.filter) {
findQuery.where = JSON.parse(req.query.filter as string);
2019-06-23 03:35:23 -07:00
if ((findQuery.where! as IDataObject).id !== undefined) {
// No idea if multiple where parameters make db search
// slower but to be sure that that is not the case we
// remove all unnecessary fields in case the id is defined.
findQuery.where = { id: (findQuery.where! as IDataObject).id };
}
}
findQuery.select = ['id', 'name', 'type', 'nodesAccess', 'createdAt', 'updatedAt'];
const results = await Db.collections.Credentials!.find(findQuery) as unknown as ICredentialsResponse[];
let encryptionKey = undefined;
const includeData = ['true', true].includes(req.query.includeData as string);
if (includeData === true) {
2019-06-23 03:35:23 -07:00
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
throw new Error('No encryption key got found to decrypt the credentials!');
}
}
let result;
for (result of results) {
(result as ICredentialsDecryptedResponse).id = result.id.toString();
}
return results;
}));
// ----------------------------------------
// Credential-Types
// ----------------------------------------
// Returns all the credential types which are defined in the loaded n8n-modules
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/credential-types`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialType[]> => {
2019-06-23 03:35:23 -07:00
const returnData: ICredentialType[] = [];
const credentialTypes = CredentialTypes();
credentialTypes.getAll().forEach((credentialData) => {
returnData.push(credentialData);
});
return returnData;
}));
// ----------------------------------------
// OAuth1-Credential/Auth
// ----------------------------------------
// Authorize OAuth Data
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/oauth1-credential/auth`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string> => {
if (req.query.id === undefined) {
res.status(500).send('Required credential id is missing!');
return '';
}
const result = await Db.collections.Credentials!.findOne(req.query.id as string);
if (result === undefined) {
res.status(404).send('The credential is not known.');
return '';
}
let encryptionKey = undefined;
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
res.status(500).send('No encryption key got found to decrypt the credentials!');
return '';
}
// Decrypt the currently saved credentials
const workflowCredentials: IWorkflowCredentials = {
[result.type as string]: {
[result.name as string]: result as ICredentialsEncrypted,
},
};
2021-01-29 00:31:40 -08:00
const mode: WorkflowExecuteMode = 'internal';
const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
2021-01-29 00:31:40 -08:00
const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true);
const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode);
const signatureMethod = _.get(oauthCredentials, 'signatureMethod') as string;
const oAuthOptions: clientOAuth1.Options = {
consumer: {
key: _.get(oauthCredentials, 'consumerKey') as string,
secret: _.get(oauthCredentials, 'consumerSecret') as string,
},
signature_method: signatureMethod,
hash_function(base, key) {
const algorithm = (signatureMethod === 'HMAC-SHA1') ? 'sha1' : 'sha256';
return createHmac(algorithm, key)
.update(base)
.digest('base64');
},
};
const oauthRequestData = {
2020-10-22 06:46:03 -07:00
oauth_callback: `${WebhookHelpers.getWebhookBaseUrl()}${this.restEndpoint}/oauth1-credential/callback?cid=${req.query.id}`,
};
await this.externalHooks.run('oauth1.authenticate', [oAuthOptions, oauthRequestData]);
const oauth = new clientOAuth1(oAuthOptions);
2020-07-14 04:59:37 -07:00
const options: RequestOptions = {
method: 'POST',
url: (_.get(oauthCredentials, 'requestTokenUrl') as string),
data: oauthRequestData,
};
const data = oauth.toHeader(oauth.authorize(options as RequestOptions));
//@ts-ignore
options.headers = data;
const response = await requestPromise(options);
// Response comes as x-www-form-urlencoded string so convert it to JSON
const responseJson = querystring.parse(response);
const returnUri = `${_.get(oauthCredentials, 'authUrl')}?oauth_token=${responseJson.oauth_token}`;
// Encrypt the data
const credentials = new Credentials(result.name, result.type, result.nodesAccess);
credentials.setData(decryptedDataOriginal, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
// Add special database related data
newCredentialsData.updatedAt = this.getCurrentDate();
// Update the credentials in DB
await Db.collections.Credentials!.update(req.query.id as string, newCredentialsData);
return returnUri;
}));
// Verify and store app code. Generate access tokens and store for respective credential.
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/oauth1-credential/callback`, async (req: express.Request, res: express.Response) => {
const { oauth_verifier, oauth_token, cid } = req.query;
if (oauth_verifier === undefined || oauth_token === undefined) {
const errorResponse = new ResponseHelper.ResponseError('Insufficient parameters for OAuth1 callback. Received following query parameters: ' + JSON.stringify(req.query), undefined, 503);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
2020-06-03 06:06:13 -07:00
const result = await Db.collections.Credentials!.findOne(cid as any); // tslint:disable-line:no-any
if (result === undefined) {
const errorResponse = new ResponseHelper.ResponseError('The credential is not known.', undefined, 404);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
let encryptionKey = undefined;
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
const errorResponse = new ResponseHelper.ResponseError('No encryption key got found to decrypt the credentials!', undefined, 503);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
// Decrypt the currently saved credentials
const workflowCredentials: IWorkflowCredentials = {
[result.type as string]: {
[result.name as string]: result as ICredentialsEncrypted,
},
};
2021-01-29 00:31:40 -08:00
const mode: WorkflowExecuteMode = 'internal';
const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
2021-01-29 00:31:40 -08:00
const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true);
const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode);
2020-07-14 04:59:37 -07:00
const options: OptionsWithUrl = {
method: 'POST',
url: _.get(oauthCredentials, 'accessTokenUrl') as string,
qs: {
oauth_token,
oauth_verifier,
2020-10-22 06:46:03 -07:00
},
};
let oauthToken;
try {
oauthToken = await requestPromise(options);
} catch (error) {
const errorResponse = new ResponseHelper.ResponseError('Unable to get access tokens!', undefined, 404);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
// Response comes as x-www-form-urlencoded string so convert it to JSON
const oauthTokenJson = querystring.parse(oauthToken);
decryptedDataOriginal.oauthTokenData = oauthTokenJson;
const credentials = new Credentials(result.name, result.type, result.nodesAccess);
credentials.setData(decryptedDataOriginal, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
// Add special database related data
newCredentialsData.updatedAt = this.getCurrentDate();
// Save the credentials in DB
2020-06-03 06:06:13 -07:00
await Db.collections.Credentials!.update(cid as any, newCredentialsData); // tslint:disable-line:no-any
res.sendFile(pathResolve(__dirname, '../../templates/oauth-callback.html'));
});
2019-06-23 03:35:23 -07:00
// ----------------------------------------
// OAuth2-Credential/Auth
// ----------------------------------------
// Authorize OAuth Data
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/oauth2-credential/auth`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string> => {
if (req.query.id === undefined) {
res.status(500).send('Required credential id is missing.');
return '';
}
const result = await Db.collections.Credentials!.findOne(req.query.id as string);
if (result === undefined) {
res.status(404).send('The credential is not known.');
return '';
}
let encryptionKey = undefined;
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
res.status(500).send('No encryption key got found to decrypt the credentials!');
return '';
}
// Decrypt the currently saved credentials
const workflowCredentials: IWorkflowCredentials = {
[result.type as string]: {
[result.name as string]: result as ICredentialsEncrypted,
},
};
2021-01-29 00:31:40 -08:00
const mode: WorkflowExecuteMode = 'internal';
const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
2021-01-29 00:31:40 -08:00
const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true);
const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode);
const token = new csrf();
// Generate a CSRF prevention token and send it as a OAuth2 state stringma/ERR
const csrfSecret = token.secretSync();
const state = {
token: token.create(csrfSecret),
cid: req.query.id,
};
const stateEncodedStr = Buffer.from(JSON.stringify(state)).toString('base64') as string;
const oAuthOptions: clientOAuth2.Options = {
clientId: _.get(oauthCredentials, 'clientId') as string,
clientSecret: _.get(oauthCredentials, 'clientSecret', '') as string,
accessTokenUri: _.get(oauthCredentials, 'accessTokenUrl', '') as string,
authorizationUri: _.get(oauthCredentials, 'authUrl', '') as string,
2020-06-23 07:12:53 -07:00
redirectUri: `${WebhookHelpers.getWebhookBaseUrl()}${this.restEndpoint}/oauth2-credential/callback`,
scopes: _.split(_.get(oauthCredentials, 'scope', 'openid,') as string, ','),
state: stateEncodedStr,
};
await this.externalHooks.run('oauth2.authenticate', [oAuthOptions]);
const oAuthObj = new clientOAuth2(oAuthOptions);
// Encrypt the data
const credentials = new Credentials(result.name, result.type, result.nodesAccess);
decryptedDataOriginal.csrfSecret = csrfSecret;
credentials.setData(decryptedDataOriginal, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
// Add special database related data
newCredentialsData.updatedAt = this.getCurrentDate();
// Update the credentials in DB
await Db.collections.Credentials!.update(req.query.id as string, newCredentialsData);
const authQueryParameters = _.get(oauthCredentials, 'authQueryParameters', '') as string;
let returnUri = oAuthObj.code.getUri();
// if scope uses comma, change it as the library always return then with spaces
if ((_.get(oauthCredentials, 'scope') as string).includes(',')) {
const data = querystring.parse(returnUri.split('?')[1] as string);
data.scope = _.get(oauthCredentials, 'scope') as string;
returnUri = `${_.get(oauthCredentials, 'authUrl', '')}?${querystring.stringify(data)}`;
}
if (authQueryParameters) {
returnUri += '&' + authQueryParameters;
}
return returnUri;
}));
2019-06-23 03:35:23 -07:00
// ----------------------------------------
// OAuth2-Credential/Callback
// ----------------------------------------
// Verify and store app code. Generate access tokens and store for respective credential.
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/oauth2-credential/callback`, async (req: express.Request, res: express.Response) => {
:sparkles: Add QuickBooks node (#1365) * Add OAuth2 credentials * Adjust credentials params * Add node to listing * Add initial node scaffolding * Remove unused credentials params * Add customer search with select statement * Add pagination to customer search * Add customer creation functionality * Add customer update functionality * Small formatting fix * Adjust property name casing for consistency * Adjust customer operations for consistency * Handle large QuickBooks listings * Add initial estimate resource description * Add estimate resource retrieval operations * Refactor customer billing address * Simplify customer billing address * Fix casing for customer additional fields * Adjust types to accommodate loadOptions functions * Add loadOptions for customers to estimate * Sort customer update fields alphabetically * Refactor estimate line into standalone file * Add stub for PDF retrieval operation * Add invoice resource description and execute branches * Implement estimate PDF download functionality * Place descriptions in their own dir * Add get and getAll for invoices * Add send functionality to invoices * Refactor handling of binary data * Add invoice voiding functionality * Add invoice deletion functionality * Refactor resources into subdirs and add interfaces * Add get and getAll for bill * Add payment description * Add get and getAll for payment * Make variables in endpoints consistent * Refactor interfaces for consistency * Add interface for item resource * Fill in fields for all resources * Minor fixes in defaults and descriptions * Refactor loader * Add all resources to execute function * Fix line property duplication * Add get and getAll for vendor * Optimize description imports * Add creation for customer and bill * Add update operation for bill * Refactor create and update for customer * Implement employee create and update * Implement create and update for estimate * Make address params more consistent * Add create and update to payment * Add item operations * Add create and delete operations for invoice * Refactor binary data handler * Refactor generic functions * Add create and update operations for vendor * Fix build * Fix total amount in bill:update * Fix balance in bill:update * Remove currency from bill:update * Implement reference retrieval in bill:update * Fix failing params in customer:update * Fix param in employee:update * Implement reference retrieval in estimate:update * Fix failing params in estimate:update * Fix failing params in invoice:update * Fix failing param in vendor:update * Implement reference retrieval in payment:update * Remove unused interfaces * Rename line property function * Remove hared directory * Refactor reference and sync token retrieval * Fix line structure in bill:create * Fix line structure in estimate:create * Flatten responses * Refactor line processing * Remove unused interfaces * Add endpoint documentation * Fix payment:void content type * Fix default for bill line item * Hide auth URI query parameters * Hide auth header parameter * Add switch for credentials environment * Adjust OAuth2 callback to accommodate realmId * Retrieve realmId from OAuth2 flow * :zap: Improvements * Reposition dividers * Add IDs to display names of reference fields * Add estimate:delete and bill:delete * Load items in lines for bill, estimate and invoice * Add filename for binary property in PDF download * :zap: Improvements * Adjust field description * Implement estimate:send * Adjust field description * Adjust custom field descriptions * Add missing period to description * :zap: Minor improvements on QuickBooks-Node * Add descriptions for bill * Add descriptions for customer * Add descriptions for employee * Add descriptions for estimate * Add descriptions for invoice * Add descriptions for payment * Add descriptions for vendor Co-authored-by: ricardo <ricardoespinoza105@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-13 08:27:08 -08:00
// realmId it's currently just use for the quickbook OAuth2 flow
const { code, state: stateEncoded } = req.query;
if (code === undefined || stateEncoded === undefined) {
const errorResponse = new ResponseHelper.ResponseError('Insufficient parameters for OAuth2 callback. Received following query parameters: ' + JSON.stringify(req.query), undefined, 503);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
let state;
try {
state = JSON.parse(Buffer.from(stateEncoded as string, 'base64').toString());
} catch (error) {
const errorResponse = new ResponseHelper.ResponseError('Invalid state format returned', undefined, 503);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
const result = await Db.collections.Credentials!.findOne(state.cid);
if (result === undefined) {
const errorResponse = new ResponseHelper.ResponseError('The credential is not known.', undefined, 404);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
let encryptionKey = undefined;
encryptionKey = await UserSettings.getEncryptionKey();
if (encryptionKey === undefined) {
const errorResponse = new ResponseHelper.ResponseError('No encryption key got found to decrypt the credentials!', undefined, 503);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
// Decrypt the currently saved credentials
const workflowCredentials: IWorkflowCredentials = {
[result.type as string]: {
[result.name as string]: result as ICredentialsEncrypted,
},
};
2021-01-29 00:31:40 -08:00
const mode: WorkflowExecuteMode = 'internal';
const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
2021-01-29 00:31:40 -08:00
const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, mode, true);
const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type, mode);
const token = new csrf();
if (decryptedDataOriginal.csrfSecret === undefined || !token.verify(decryptedDataOriginal.csrfSecret as string, state.token)) {
const errorResponse = new ResponseHelper.ResponseError('The OAuth2 callback state is invalid!', undefined, 404);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
2020-02-12 07:04:43 -08:00
let options = {};
2020-07-08 12:15:25 -07:00
const oAuth2Parameters = {
clientId: _.get(oauthCredentials, 'clientId') as string,
2020-09-23 00:28:02 -07:00
clientSecret: _.get(oauthCredentials, 'clientSecret', '') as string | undefined,
2020-07-08 12:15:25 -07:00
accessTokenUri: _.get(oauthCredentials, 'accessTokenUrl', '') as string,
authorizationUri: _.get(oauthCredentials, 'authUrl', '') as string,
redirectUri: `${WebhookHelpers.getWebhookBaseUrl()}${this.restEndpoint}/oauth2-credential/callback`,
2020-10-22 06:46:03 -07:00
scopes: _.split(_.get(oauthCredentials, 'scope', 'openid,') as string, ','),
2020-07-08 12:15:25 -07:00
};
if (_.get(oauthCredentials, 'authentication', 'header') as string === 'body') {
options = {
body: {
client_id: _.get(oauthCredentials, 'clientId') as string,
client_secret: _.get(oauthCredentials, 'clientSecret', '') as string,
},
};
2020-07-08 12:15:25 -07:00
delete oAuth2Parameters.clientSecret;
}
await this.externalHooks.run('oauth2.callback', [oAuth2Parameters]);
2020-02-12 07:04:43 -08:00
2020-07-08 12:15:25 -07:00
const oAuthObj = new clientOAuth2(oAuth2Parameters);
const queryParameters = req.originalUrl.split('?').splice(1, 1).join('');
const oauthToken = await oAuthObj.code.getToken(`${oAuth2Parameters.redirectUri}?${queryParameters}`, options);
:sparkles: Add QuickBooks node (#1365) * Add OAuth2 credentials * Adjust credentials params * Add node to listing * Add initial node scaffolding * Remove unused credentials params * Add customer search with select statement * Add pagination to customer search * Add customer creation functionality * Add customer update functionality * Small formatting fix * Adjust property name casing for consistency * Adjust customer operations for consistency * Handle large QuickBooks listings * Add initial estimate resource description * Add estimate resource retrieval operations * Refactor customer billing address * Simplify customer billing address * Fix casing for customer additional fields * Adjust types to accommodate loadOptions functions * Add loadOptions for customers to estimate * Sort customer update fields alphabetically * Refactor estimate line into standalone file * Add stub for PDF retrieval operation * Add invoice resource description and execute branches * Implement estimate PDF download functionality * Place descriptions in their own dir * Add get and getAll for invoices * Add send functionality to invoices * Refactor handling of binary data * Add invoice voiding functionality * Add invoice deletion functionality * Refactor resources into subdirs and add interfaces * Add get and getAll for bill * Add payment description * Add get and getAll for payment * Make variables in endpoints consistent * Refactor interfaces for consistency * Add interface for item resource * Fill in fields for all resources * Minor fixes in defaults and descriptions * Refactor loader * Add all resources to execute function * Fix line property duplication * Add get and getAll for vendor * Optimize description imports * Add creation for customer and bill * Add update operation for bill * Refactor create and update for customer * Implement employee create and update * Implement create and update for estimate * Make address params more consistent * Add create and update to payment * Add item operations * Add create and delete operations for invoice * Refactor binary data handler * Refactor generic functions * Add create and update operations for vendor * Fix build * Fix total amount in bill:update * Fix balance in bill:update * Remove currency from bill:update * Implement reference retrieval in bill:update * Fix failing params in customer:update * Fix param in employee:update * Implement reference retrieval in estimate:update * Fix failing params in estimate:update * Fix failing params in invoice:update * Fix failing param in vendor:update * Implement reference retrieval in payment:update * Remove unused interfaces * Rename line property function * Remove hared directory * Refactor reference and sync token retrieval * Fix line structure in bill:create * Fix line structure in estimate:create * Flatten responses * Refactor line processing * Remove unused interfaces * Add endpoint documentation * Fix payment:void content type * Fix default for bill line item * Hide auth URI query parameters * Hide auth header parameter * Add switch for credentials environment * Adjust OAuth2 callback to accommodate realmId * Retrieve realmId from OAuth2 flow * :zap: Improvements * Reposition dividers * Add IDs to display names of reference fields * Add estimate:delete and bill:delete * Load items in lines for bill, estimate and invoice * Add filename for binary property in PDF download * :zap: Improvements * Adjust field description * Implement estimate:send * Adjust field description * Adjust custom field descriptions * Add missing period to description * :zap: Minor improvements on QuickBooks-Node * Add descriptions for bill * Add descriptions for customer * Add descriptions for employee * Add descriptions for estimate * Add descriptions for invoice * Add descriptions for payment * Add descriptions for vendor Co-authored-by: ricardo <ricardoespinoza105@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-13 08:27:08 -08:00
if (Object.keys(req.query).length > 2) {
_.set(oauthToken.data, 'callbackQueryString', _.omit(req.query, 'state', 'code'));
}
if (oauthToken === undefined) {
const errorResponse = new ResponseHelper.ResponseError('Unable to get access tokens!', undefined, 404);
return ResponseHelper.sendErrorResponse(res, errorResponse);
}
if (decryptedDataOriginal.oauthTokenData) {
// Only overwrite supplied data as some providers do for example just return the
// refresh_token on the very first request and not on subsequent ones.
Object.assign(decryptedDataOriginal.oauthTokenData, oauthToken.data);
} else {
// No data exists so simply set
decryptedDataOriginal.oauthTokenData = oauthToken.data;
}
_.unset(decryptedDataOriginal, 'csrfSecret');
const credentials = new Credentials(result.name, result.type, result.nodesAccess);
credentials.setData(decryptedDataOriginal, encryptionKey);
const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
// Add special database related data
newCredentialsData.updatedAt = this.getCurrentDate();
// Save the credentials in DB
await Db.collections.Credentials!.update(state.cid, newCredentialsData);
res.sendFile(pathResolve(__dirname, '../../templates/oauth-callback.html'));
});
2019-06-23 03:35:23 -07:00
// ----------------------------------------
// Executions
// ----------------------------------------
// Returns all finished executions
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/executions`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IExecutionsListResponse> => {
2019-06-23 03:35:23 -07:00
let filter: any = {}; // tslint:disable-line:no-any
if (req.query.filter) {
filter = JSON.parse(req.query.filter as string);
2019-06-23 03:35:23 -07:00
}
let limit = 20;
if (req.query.limit) {
limit = parseInt(req.query.limit as string, 10);
2019-06-23 03:35:23 -07:00
}
const executingWorkflowIds: string[] = [];
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (config.get('executions.mode') === 'queue') {
const currentJobs = await Queue.getInstance().getJobs(['active', 'waiting']);
executingWorkflowIds.push(...currentJobs.map(job => job.data.executionId) as string[]);
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
}
// We may have manual executions even with queue so we must account for these.
executingWorkflowIds.push(...this.activeExecutionsInstance.getActiveExecutions().map(execution => execution.id.toString()) as string[]);
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
2019-06-23 03:35:23 -07:00
const countFilter = JSON.parse(JSON.stringify(filter));
countFilter.select = ['id'];
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
countFilter.where = {id: Not(In(executingWorkflowIds))};
const resultsQuery = await Db.collections.Execution!
.createQueryBuilder("execution")
.select([
'execution.id',
'execution.finished',
'execution.mode',
'execution.retryOf',
'execution.retrySuccessId',
'execution.startedAt',
'execution.stoppedAt',
'execution.workflowData',
])
.orderBy('execution.id', 'DESC')
.take(limit);
Object.keys(filter).forEach((filterField) => {
resultsQuery.andWhere(`execution.${filterField} = :${filterField}`, {[filterField]: filter[filterField]});
2019-06-23 03:35:23 -07:00
});
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
if (req.query.lastId) {
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
resultsQuery.andWhere(`execution.id < :lastId`, {lastId: req.query.lastId});
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
}
if (req.query.firstId) {
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
resultsQuery.andWhere(`execution.id > :firstId`, {firstId: req.query.firstId});
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
}
if (executingWorkflowIds.length > 0) {
resultsQuery.andWhere(`execution.id NOT IN (:...ids)`, {ids: executingWorkflowIds});
}
2019-06-23 03:35:23 -07:00
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
const resultsPromise = resultsQuery.getMany();
2019-06-23 03:35:23 -07:00
const countPromise = Db.collections.Execution!.count(countFilter);
const results: IExecutionFlattedDb[] = await resultsPromise;
const count = await countPromise;
const returnResults: IExecutionsSummary[] = [];
for (const result of results) {
returnResults.push({
id: result.id!.toString(),
finished: result.finished,
mode: result.mode,
retryOf: result.retryOf ? result.retryOf.toString() : undefined,
retrySuccessId: result.retrySuccessId ? result.retrySuccessId.toString() : undefined,
startedAt: result.startedAt,
stoppedAt: result.stoppedAt,
workflowId: result.workflowData!.id ? result.workflowData!.id!.toString() : '',
2019-06-23 03:35:23 -07:00
workflowName: result.workflowData!.name,
});
}
return {
count,
results: returnResults,
};
}));
// Returns a specific execution
this.app.get(`/${this.restEndpoint}/executions/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IExecutionResponse | IExecutionFlattedResponse | undefined> => {
2019-06-23 03:35:23 -07:00
const result = await Db.collections.Execution!.findOne(req.params.id);
if (result === undefined) {
return undefined;
}
2020-10-28 15:38:53 -07:00
if (req.query.unflattedResponse === 'true') {
:sparkles: Add QuickBooks node (#1365) * Add OAuth2 credentials * Adjust credentials params * Add node to listing * Add initial node scaffolding * Remove unused credentials params * Add customer search with select statement * Add pagination to customer search * Add customer creation functionality * Add customer update functionality * Small formatting fix * Adjust property name casing for consistency * Adjust customer operations for consistency * Handle large QuickBooks listings * Add initial estimate resource description * Add estimate resource retrieval operations * Refactor customer billing address * Simplify customer billing address * Fix casing for customer additional fields * Adjust types to accommodate loadOptions functions * Add loadOptions for customers to estimate * Sort customer update fields alphabetically * Refactor estimate line into standalone file * Add stub for PDF retrieval operation * Add invoice resource description and execute branches * Implement estimate PDF download functionality * Place descriptions in their own dir * Add get and getAll for invoices * Add send functionality to invoices * Refactor handling of binary data * Add invoice voiding functionality * Add invoice deletion functionality * Refactor resources into subdirs and add interfaces * Add get and getAll for bill * Add payment description * Add get and getAll for payment * Make variables in endpoints consistent * Refactor interfaces for consistency * Add interface for item resource * Fill in fields for all resources * Minor fixes in defaults and descriptions * Refactor loader * Add all resources to execute function * Fix line property duplication * Add get and getAll for vendor * Optimize description imports * Add creation for customer and bill * Add update operation for bill * Refactor create and update for customer * Implement employee create and update * Implement create and update for estimate * Make address params more consistent * Add create and update to payment * Add item operations * Add create and delete operations for invoice * Refactor binary data handler * Refactor generic functions * Add create and update operations for vendor * Fix build * Fix total amount in bill:update * Fix balance in bill:update * Remove currency from bill:update * Implement reference retrieval in bill:update * Fix failing params in customer:update * Fix param in employee:update * Implement reference retrieval in estimate:update * Fix failing params in estimate:update * Fix failing params in invoice:update * Fix failing param in vendor:update * Implement reference retrieval in payment:update * Remove unused interfaces * Rename line property function * Remove hared directory * Refactor reference and sync token retrieval * Fix line structure in bill:create * Fix line structure in estimate:create * Flatten responses * Refactor line processing * Remove unused interfaces * Add endpoint documentation * Fix payment:void content type * Fix default for bill line item * Hide auth URI query parameters * Hide auth header parameter * Add switch for credentials environment * Adjust OAuth2 callback to accommodate realmId * Retrieve realmId from OAuth2 flow * :zap: Improvements * Reposition dividers * Add IDs to display names of reference fields * Add estimate:delete and bill:delete * Load items in lines for bill, estimate and invoice * Add filename for binary property in PDF download * :zap: Improvements * Adjust field description * Implement estimate:send * Adjust field description * Adjust custom field descriptions * Add missing period to description * :zap: Minor improvements on QuickBooks-Node * Add descriptions for bill * Add descriptions for customer * Add descriptions for employee * Add descriptions for estimate * Add descriptions for invoice * Add descriptions for payment * Add descriptions for vendor Co-authored-by: ricardo <ricardoespinoza105@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-13 08:27:08 -08:00
const fullExecutionData = ResponseHelper.unflattenExecutionData(result);
return fullExecutionData as IExecutionResponse;
} else {
// Convert to response format in which the id is a string
(result as IExecutionFlatted as IExecutionFlattedResponse).id = result.id.toString();
return result as IExecutionFlatted as IExecutionFlattedResponse;
}
2019-06-23 03:35:23 -07:00
}));
// Retries a failed execution
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/executions/:id/retry`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
2019-06-23 03:35:23 -07:00
// Get the data to execute
const fullExecutionDataFlatted = await Db.collections.Execution!.findOne(req.params.id);
if (fullExecutionDataFlatted === undefined) {
2019-08-28 08:16:09 -07:00
throw new ResponseHelper.ResponseError(`The execution with the id "${req.params.id}" does not exist.`, 404, 404);
2019-06-23 03:35:23 -07:00
}
const fullExecutionData = ResponseHelper.unflattenExecutionData(fullExecutionDataFlatted);
if (fullExecutionData.finished === true) {
throw new Error('The execution did succeed and can so not be retried.');
}
const executionMode = 'retry';
const credentials = await WorkflowCredentials(fullExecutionData.workflowData.nodes);
fullExecutionData.workflowData.active = false;
2019-06-23 03:35:23 -07:00
// Start the workflow
const data: IWorkflowExecutionDataProcess = {
credentials,
executionMode,
executionData: fullExecutionData.data,
retryOf: req.params.id,
workflowData: fullExecutionData.workflowData,
};
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
const lastNodeExecuted = data!.executionData!.resultData.lastNodeExecuted as string | undefined;
if (lastNodeExecuted) {
// Remove the old error and the data of the last run of the node that it can be replaced
delete data!.executionData!.resultData.error;
const length = data!.executionData!.resultData.runData[lastNodeExecuted].length;
if (length > 0 && data!.executionData!.resultData.runData[lastNodeExecuted][length - 1].error !== undefined) {
// Remove results only if it is an error.
:sparkles: Unify execution id + Queue system (#1340) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * :zap: Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-08 23:59:32 -08:00
// If we are retrying due to a crash, the information is simply success info from last node
data!.executionData!.resultData.runData[lastNodeExecuted].pop();
// Stack will determine what to run next
}
}
2020-03-17 01:07:12 -07:00
if (req.body.loadWorkflow === true) {
// Loads the currently saved workflow to execute instead of the
// one saved at the time of the execution.
const workflowId = fullExecutionData.workflowData.id;
const workflowData = await Db.collections.Workflow!.findOne(workflowId) as IWorkflowBase;
if (workflowData === undefined) {
throw new Error(`The workflow with the ID "${workflowId}" could not be found and so the data not be loaded for the retry.`);
}
2020-03-17 01:07:12 -07:00
data.workflowData = workflowData;
const nodeTypes = NodeTypes();
const workflowInstance = new Workflow({ id: workflowData.id as string, name: workflowData.name, nodes: workflowData.nodes, connections: workflowData.connections, active: false, nodeTypes, staticData: undefined, settings: workflowData.settings });
2020-03-17 01:07:12 -07:00
// Replace all of the nodes in the execution stack with the ones of the new workflow
for (const stack of data!.executionData!.executionData!.nodeExecutionStack) {
// Find the data of the last executed node in the new workflow
const node = workflowInstance.getNode(stack.node.name);
if (node === null) {
2020-03-17 01:07:12 -07:00
throw new Error(`Could not find the node "${stack.node.name}" in workflow. It probably got deleted or renamed. Without it the workflow can sadly not be retried.`);
}
// Replace the node data in the stack that it really uses the current data
stack.node = node;
}
}
const workflowRunner = new WorkflowRunner();
const executionId = await workflowRunner.run(data);
2019-06-23 03:35:23 -07:00
2019-08-08 22:37:10 -07:00
const executionData = await this.activeExecutionsInstance.getPostExecutePromise(executionId);
if (executionData === undefined) {
throw new Error('The retry did not start for an unknown reason.');
}
return !!executionData.finished;
2019-06-23 03:35:23 -07:00
}));
// Delete Executions
// INFORMATION: We use POST instead of DELETE to not run into any issues
// with the query data getting to long
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/executions/delete`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<void> => {
2019-06-23 03:35:23 -07:00
const deleteData = req.body as IExecutionDeleteFilter;
if (deleteData.deleteBefore !== undefined) {
const filters = {
startedAt: LessThanOrEqual(deleteData.deleteBefore),
};
if (deleteData.filters !== undefined) {
Object.assign(filters, deleteData.filters);
}
await Db.collections.Execution!.delete(filters);
} else if (deleteData.ids !== undefined) {
// Deletes all executions with the given ids
await Db.collections.Execution!.delete(deleteData.ids);
} else {
throw new Error('Required body-data "ids" or "deleteBefore" is missing!');
}
}));
// ----------------------------------------
// Executing Workflows
// ----------------------------------------
// Returns all the currently working executions
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/executions-current`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IExecutionsSummary[]> => {
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (config.get('executions.mode') === 'queue') {
const currentJobs = await Queue.getInstance().getJobs(['active', 'waiting']);
const currentlyRunningQueueIds = currentJobs.map(job => job.data.executionId);
const currentlyRunningManualExecutions = this.activeExecutionsInstance.getActiveExecutions();
const manualExecutionIds = currentlyRunningManualExecutions.map(execution => execution.id);
const currentlyRunningExecutionIds = currentlyRunningQueueIds.concat(manualExecutionIds);
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (currentlyRunningExecutionIds.length === 0) {
return [];
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const resultsQuery = await Db.collections.Execution!
.createQueryBuilder("execution")
.select([
'execution.id',
'execution.workflowId',
'execution.mode',
'execution.retryOf',
'execution.startedAt',
])
.orderBy('execution.id', 'DESC')
.andWhere(`execution.id IN (:...ids)`, {ids: currentlyRunningExecutionIds});
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (req.query.filter) {
const filter = JSON.parse(req.query.filter as string);
if (filter.workflowId !== undefined) {
resultsQuery.andWhere('execution.workflowId = :workflowId', {workflowId: filter.workflowId});
}
}
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const results = await resultsQuery.getMany();
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
return results.map(result => {
return {
id: result.id,
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
workflowId: result.workflowId,
mode: result.mode,
retryOf: result.retryOf !== null ? result.retryOf : undefined,
startedAt: new Date(result.startedAt),
} as IExecutionsSummary;
});
} else {
const executingWorkflows = this.activeExecutionsInstance.getActiveExecutions();
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const returnData: IExecutionsSummary[] = [];
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
let filter: any = {}; // tslint:disable-line:no-any
if (req.query.filter) {
filter = JSON.parse(req.query.filter as string);
2019-06-23 03:35:23 -07:00
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
for (const data of executingWorkflows) {
if (filter.workflowId !== undefined && filter.workflowId !== data.workflowId) {
continue;
2019-06-23 03:35:23 -07:00
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
returnData.push(
{
id: data.id.toString(),
workflowId: data.workflowId === undefined ? '' : data.workflowId.toString(),
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
mode: data.mode,
retryOf: data.retryOf,
startedAt: new Date(data.startedAt),
}
);
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
return returnData;
2019-06-23 03:35:23 -07:00
}
}));
// Forces the execution to stop
2020-06-23 07:12:53 -07:00
this.app.post(`/${this.restEndpoint}/executions-current/:id/stop`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IExecutionsStopData> => {
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (config.get('executions.mode') === 'queue') {
// Manual executions should still be stoppable, so
// try notifying the `activeExecutions` to stop it.
const result = await this.activeExecutionsInstance.stopExecution(req.params.id);
if (result !== undefined) {
const returnData: IExecutionsStopData = {
mode: result.mode,
startedAt: new Date(result.startedAt),
stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined,
finished: result.finished,
};
return returnData;
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const currentJobs = await Queue.getInstance().getJobs(['active', 'waiting']);
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const job = currentJobs.find(job => job.data.executionId.toString() === req.params.id);
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (!job) {
throw new Error(`Could not stop "${req.params.id}" as it is no longer in queue.`);
} else {
await Queue.getInstance().stopJob(job);
}
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const executionDb = await Db.collections.Execution?.findOne(req.params.id) as IExecutionFlattedDb;
const fullExecutionData = ResponseHelper.unflattenExecutionData(executionDb) as IExecutionResponse;
2019-06-23 03:35:23 -07:00
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const returnData: IExecutionsStopData = {
mode: fullExecutionData.mode,
startedAt: new Date(fullExecutionData.startedAt),
stoppedAt: fullExecutionData.stoppedAt ? new Date(fullExecutionData.stoppedAt) : undefined,
finished: fullExecutionData.finished,
};
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
return returnData;
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
} else {
const executionId = req.params.id;
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
// Stopt he execution and wait till it is done and we got the data
const result = await this.activeExecutionsInstance.stopExecution(executionId);
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (result === undefined) {
throw new Error(`The execution id "${executionId}" could not be found.`);
}
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
const returnData: IExecutionsStopData = {
mode: result.mode,
startedAt: new Date(result.startedAt),
stoppedAt: result.stoppedAt ? new Date(result.stoppedAt) : undefined,
finished: result.finished,
};
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
return returnData;
}
2019-06-23 03:35:23 -07:00
}));
// Removes a test webhook
2020-06-23 07:12:53 -07:00
this.app.delete(`/${this.restEndpoint}/test-webhook/:id`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
2019-06-23 03:35:23 -07:00
const workflowId = req.params.id;
return this.testWebhooks.cancelTestWebhook(workflowId);
}));
// ----------------------------------------
// Options
// ----------------------------------------
// Returns all the available timezones
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/options/timezones`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<object> => {
2019-06-23 03:35:23 -07:00
return timezones;
}));
// ----------------------------------------
// Settings
// ----------------------------------------
// Returns the settings which are needed in the UI
2020-06-23 07:12:53 -07:00
this.app.get(`/${this.restEndpoint}/settings`, ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IN8nUISettings> => {
return this.frontendSettings;
2019-06-23 03:35:23 -07:00
}));
// ----------------------------------------
// Webhooks
// ----------------------------------------
:sparkles: Separate webhooks from core (#1408) * Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * :sparkles: Add bull to execute workflows * :shirt: Fix lint issue * :zap: Add graceful shutdown to worker * :zap: Add loading staticData to worker * :shirt: Fix lint issue * :zap: Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * Added webhooks service and setting to disable webhooks from main process * Fixed executions list when running with queues. Now we get the list of actively running workflows from bull. * Add option to disable deregistration of webhooks on shutdown * Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat. * Added auto refresh to executions list * Improvements to workflow stop process when running with queues * Refactor queue system to use a singleton and avoid code duplication * Improve comments and remove unnecessary commits * Remove console.log from vue file * Blocking webhook process to run without queues * Handling execution stop graciously when possible * Removing initialization of all workflows from webhook process * Refactoring code to remove code duplication for job stop * Improved execution list to be more fluid and less intrusive * Fixing workflow name for current executions when auto updating * :zap: Right align autorefresh checkbox Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2021-02-09 14:32:40 -08:00
if (config.get('endpoints.disableProductionWebhooksOnMainProcess') !== true) {
WebhookServer.registerProductionWebhooks.apply(this);
}
2019-06-23 03:35:23 -07:00
2020-04-26 10:51:20 -07:00
// HEAD webhook requests (test for UI)
this.app.head(`/${this.endpointWebhookTest}/*`, async (req: express.Request, res: express.Response) => {
// Cut away the "/webhook-test/" to get the registred part of the url
const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookTest.length + 2);
let response;
try {
response = await this.testWebhooks.callTestWebhook('HEAD', requestUrl, req, res);
} catch (error) {
ResponseHelper.sendErrorResponse(res, error);
return;
}
if (response.noWebhookResponse === true) {
// Nothing else to do as the response got already sent
return;
}
ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
});
2019-06-23 03:35:23 -07:00
// HEAD webhook requests (test for UI)
this.app.options(`/${this.endpointWebhookTest}/*`, async (req: express.Request, res: express.Response) => {
// Cut away the "/webhook-test/" to get the registred part of the url
const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookTest.length + 2);
let allowedMethods: string[];
try {
allowedMethods = await this.testWebhooks.getWebhookMethods(requestUrl);
allowedMethods.push('OPTIONS');
// Add custom "Allow" header to satisfy OPTIONS response.
res.append('Allow', allowedMethods);
} catch (error) {
ResponseHelper.sendErrorResponse(res, error);
return;
}
ResponseHelper.sendSuccessResponse(res, {}, true, 204);
});
2019-06-23 03:35:23 -07:00
// GET webhook requests (test for UI)
this.app.get(`/${this.endpointWebhookTest}/*`, async (req: express.Request, res: express.Response) => {
// Cut away the "/webhook-test/" to get the registred part of the url
const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookTest.length + 2);
let response;
try {
response = await this.testWebhooks.callTestWebhook('GET', requestUrl, req, res);
} catch (error) {
ResponseHelper.sendErrorResponse(res, error);
return;
}
if (response.noWebhookResponse === true) {
// Nothing else to do as the response got already sent
return;
}
ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
2019-06-23 03:35:23 -07:00
});
// POST webhook requests (test for UI)
this.app.post(`/${this.endpointWebhookTest}/*`, async (req: express.Request, res: express.Response) => {
// Cut away the "/webhook-test/" to get the registred part of the url
const requestUrl = (req as ICustomRequest).parsedUrl!.pathname!.slice(this.endpointWebhookTest.length + 2);
let response;
try {
response = await this.testWebhooks.callTestWebhook('POST', requestUrl, req, res);
} catch (error) {
ResponseHelper.sendErrorResponse(res, error);
return;
}
if (response.noWebhookResponse === true) {
// Nothing else to do as the response got already sent
return;
}
ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
2019-06-23 03:35:23 -07:00
});
if (this.endpointPresetCredentials !== '') {
// POST endpoint to set preset credentials
this.app.post(`/${this.endpointPresetCredentials}`, async (req: express.Request, res: express.Response) => {
if (this.presetCredentialsLoaded === false) {
const body = req.body as ICredentialsOverwrite;
if (req.headers['content-type'] !== 'application/json') {
ResponseHelper.sendErrorResponse(res, new Error('Body must be a valid JSON, make sure the content-type is application/json'));
return;
}
const loadNodesAndCredentials = LoadNodesAndCredentials();
const credentialsOverwrites = CredentialsOverwrites();
await credentialsOverwrites.init(body);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
this.presetCredentialsLoaded = true;
ResponseHelper.sendSuccessResponse(res, { success: true }, true, 200);
} else {
ResponseHelper.sendErrorResponse(res, new Error('Preset credentials can be set once'));
}
});
}
// Read the index file and replace the path placeholder
const editorUiPath = require.resolve('n8n-editor-ui');
const filePath = pathJoin(pathDirname(editorUiPath), 'dist', 'index.html');
const n8nPath = config.get('path');
2020-07-14 14:36:05 -07:00
let readIndexFile = readFileSync(filePath, 'utf8');
readIndexFile = readIndexFile.replace(/\/%BASE_PATH%\//g, n8nPath);
2020-08-14 14:20:07 -07:00
readIndexFile = readIndexFile.replace(/\/favicon.ico/g, `${n8nPath}favicon.ico`);
// Serve the altered index.html file separately
this.app.get(`/index.html`, async (req: express.Request, res: express.Response) => {
res.send(readIndexFile);
});
2019-06-23 03:35:23 -07:00
// Serve the website
const startTime = (new Date()).toUTCString();
this.app.use('/', express.static(pathJoin(pathDirname(editorUiPath), 'dist'), {
index: 'index.html',
setHeaders: (res, path) => {
if (res.req && res.req.url === '/index.html') {
// Set last modified date manually to n8n start time so
// that it hopefully refreshes the page when a new version
// got used
res.setHeader('Last-Modified', startTime);
}
2020-10-22 06:46:03 -07:00
},
}));
2019-06-23 03:35:23 -07:00
}
}
export async function start(): Promise<void> {
const PORT = config.get('port');
const ADDRESS = config.get('listen_address');
2019-06-23 03:35:23 -07:00
const app = new App();
await app.config();
2019-06-23 03:35:23 -07:00
let server;
if (app.protocol === 'https' && app.sslKey && app.sslCert) {
const https = require('https');
const privateKey = readFileSync(app.sslKey, 'utf8');
const cert = readFileSync(app.sslCert, 'utf8');
const credentials = { key: privateKey, cert };
server = https.createServer(credentials, app.app);
} else {
const http = require('http');
server = http.createServer(app.app);
}
2019-06-23 03:35:23 -07:00
server.listen(PORT, ADDRESS, async () => {
const versions = await GenericHelpers.getVersions();
console.log(`n8n ready on ${ADDRESS}, port ${PORT}`);
console.log(`Version: ${versions.cli}`);
await app.externalHooks.run('n8n.ready', [app]);
2019-06-23 03:35:23 -07:00
});
}