mirror of
https://github.com/n8n-io/n8n.git
synced 2025-01-11 21:07:28 -08:00
feat(core): Cache test webhook registrations (#8176)
In a multi-main setup, we have the following issue. The user's client connects to main A and runs a test webhook, so main A starts listening for a webhook call. A third-party service sends a request to the test webhook URL. The request is forwarded by the load balancer to main B, who is not listening for this webhook call. Therefore, the webhook call is unhandled. To start addressing this, cache test webhook registrations, using Redis for queue mode and in-memory for regular mode. When the third-party service sends a request to the test webhook URL, the request is forwarded by the load balancer to main B, who fetches test webhooks from the cache and, if it finds a match, executes the test webhook. This should be transparent - test webhook behavior should remain the same as so far. Notes: - Test webhook timeouts are not cached. A timeout is only relevant to the process it was created in, so another process retrieving from Redis a "foreign" timeout will be unable to act on it. A timeout also has circular references, so `cache-manager-ioredis-yet` is unable to serialize it. - In a single-main scenario, the timeout remains in the single process and is cleared on test webhook expiration, successful execution, and manual cancellation - all as usual. - In a multi-main scenario, we will need to have the process who received the webhook call send a message to the process who created the webhook directing this originating process to clear the timeout. This will likely be implemented via execution lifecycle hooks and Redis channel messages checking session ID. This implementation is out of scope for this PR and will come next. - Additional data in test webhooks is not cached. From what I can tell, additional data is not needed for test webhooks to be executed. Additional data also has circular references, so `cache-manager-ioredis-yet` is unable to serialize it. Follow-up to: #8155
This commit is contained in:
Iván Ovejero
GitHub
parent
053503531f
commit
22a5f5258d
|
|
@ -23,7 +23,6 @@ import type {
|
|||
INodeProperties,
|
||||
IUserSettings,
|
||||
IHttpRequestMethods,
|
||||
IWebhookData,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
|
||||
|
|
@ -743,12 +742,3 @@ export abstract class SecretsProvider {
|
|||
}
|
||||
|
||||
export type N8nInstanceType = 'main' | 'webhook' | 'worker';
|
||||
|
||||
export type WebhookRegistration = {
|
||||
sessionId?: string;
|
||||
timeout: NodeJS.Timeout;
|
||||
workflowEntity: IWorkflowDb;
|
||||
workflow: Workflow;
|
||||
destinationNode?: string;
|
||||
webhook: IWebhookData;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,53 +1,39 @@
|
|||
import type express from 'express';
|
||||
import { Service } from 'typedi';
|
||||
|
||||
import {
|
||||
type IWebhookData,
|
||||
type IWorkflowExecuteAdditionalData,
|
||||
type IHttpRequestMethods,
|
||||
type Workflow,
|
||||
type WorkflowActivateMode,
|
||||
type WorkflowExecuteMode,
|
||||
WebhookPathTakenError,
|
||||
Workflow,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import type {
|
||||
IResponseCallbackData,
|
||||
IWebhookManager,
|
||||
IWorkflowDb,
|
||||
WebhookRegistration,
|
||||
WebhookAccessControlOptions,
|
||||
WebhookRequest,
|
||||
} from '@/Interfaces';
|
||||
import { Push } from '@/push';
|
||||
import { NodeTypes } from '@/NodeTypes';
|
||||
import * as WebhookHelpers from '@/WebhookHelpers';
|
||||
import { NotFoundError } from './errors/response-errors/not-found.error';
|
||||
import { TIME } from './constants';
|
||||
import { WorkflowMissingIdError } from './errors/workflow-missing-id.error';
|
||||
import { WebhookNotFoundError } from './errors/response-errors/webhook-not-found.error';
|
||||
import { TIME } from '@/constants';
|
||||
import { NotFoundError } from '@/errors/response-errors/not-found.error';
|
||||
import { WorkflowMissingIdError } from '@/errors/workflow-missing-id.error';
|
||||
import { WebhookNotFoundError } from '@/errors/response-errors/webhook-not-found.error';
|
||||
import * as NodeExecuteFunctions from 'n8n-core';
|
||||
import { removeTrailingSlash } from './utils';
|
||||
import { TestWebhookRegistrationsService } from '@/services/test-webhook-registrations.service';
|
||||
|
||||
@Service()
|
||||
export class TestWebhooks implements IWebhookManager {
|
||||
constructor(
|
||||
private readonly push: Push,
|
||||
private readonly nodeTypes: NodeTypes,
|
||||
private readonly registrations: TestWebhookRegistrationsService,
|
||||
) {}
|
||||
|
||||
private registrations: { [webhookKey: string]: WebhookRegistration } = {};
|
||||
|
||||
private get webhooksByWorkflow() {
|
||||
const result: { [workflowId: string]: IWebhookData[] } = {};
|
||||
|
||||
for (const registration of Object.values(this.registrations)) {
|
||||
result[registration.webhook.workflowId] ||= [];
|
||||
result[registration.webhook.workflowId].push(registration.webhook);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
private timeouts: { [webhookKey: string]: NodeJS.Timeout } = {};
|
||||
|
||||
/**
|
||||
* Return a promise that resolves when the test webhook is called.
|
||||
|
|
@ -63,7 +49,7 @@ export class TestWebhooks implements IWebhookManager {
|
|||
|
||||
request.params = {} as WebhookRequest['params'];
|
||||
|
||||
let webhook = this.getActiveWebhook(httpMethod, path);
|
||||
let webhook = await this.getActiveWebhook(httpMethod, path);
|
||||
|
||||
if (!webhook) {
|
||||
// no static webhook, so check if dynamic
|
||||
|
|
@ -71,7 +57,7 @@ export class TestWebhooks implements IWebhookManager {
|
|||
|
||||
const [webhookId, ...segments] = path.split('/');
|
||||
|
||||
webhook = this.getActiveWebhook(httpMethod, segments.join('/'), webhookId);
|
||||
webhook = await this.getActiveWebhook(httpMethod, segments.join('/'), webhookId);
|
||||
|
||||
if (!webhook)
|
||||
throw new WebhookNotFoundError({
|
||||
|
|
@ -89,17 +75,22 @@ export class TestWebhooks implements IWebhookManager {
|
|||
});
|
||||
}
|
||||
|
||||
const key = this.toWebhookKey(webhook);
|
||||
const key = this.registrations.toKey(webhook);
|
||||
|
||||
if (!this.registrations[key])
|
||||
const registration = await this.registrations.get(key);
|
||||
|
||||
if (!registration) {
|
||||
throw new WebhookNotFoundError({
|
||||
path,
|
||||
httpMethod,
|
||||
webhookMethods: await this.getWebhookMethods(path),
|
||||
});
|
||||
}
|
||||
|
||||
const { destinationNode, sessionId, workflow, workflowEntity, timeout } =
|
||||
this.registrations[key];
|
||||
const { destinationNode, sessionId, workflowEntity } = registration;
|
||||
const timeout = this.timeouts[key];
|
||||
|
||||
const workflow = this.toWorkflow(workflowEntity);
|
||||
|
||||
const workflowStartNode = workflow.getNode(webhook.node);
|
||||
|
||||
|
|
@ -117,8 +108,8 @@ export class TestWebhooks implements IWebhookManager {
|
|||
workflowStartNode,
|
||||
executionMode,
|
||||
sessionId,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined, // IRunExecutionData
|
||||
undefined, // executionId
|
||||
request,
|
||||
response,
|
||||
(error: Error | null, data: IResponseCallbackData) => {
|
||||
|
|
@ -145,14 +136,17 @@ export class TestWebhooks implements IWebhookManager {
|
|||
|
||||
// Delete webhook also if an error is thrown
|
||||
if (timeout) clearTimeout(timeout);
|
||||
delete this.registrations[key];
|
||||
|
||||
await this.registrations.deregisterAll();
|
||||
|
||||
await this.deactivateWebhooks(workflow);
|
||||
});
|
||||
}
|
||||
|
||||
async getWebhookMethods(path: string) {
|
||||
const webhookMethods = Object.keys(this.registrations)
|
||||
const allKeys = await this.registrations.getAllKeys();
|
||||
|
||||
const webhookMethods = allKeys
|
||||
.filter((key) => key.includes(path))
|
||||
.map((key) => key.split('|')[0] as IHttpRequestMethods);
|
||||
|
||||
|
|
@ -162,13 +156,20 @@ export class TestWebhooks implements IWebhookManager {
|
|||
}
|
||||
|
||||
async findAccessControlOptions(path: string, httpMethod: IHttpRequestMethods) {
|
||||
const webhookKey = Object.keys(this.registrations).find(
|
||||
(key) => key.includes(path) && key.startsWith(httpMethod),
|
||||
);
|
||||
const allKeys = await this.registrations.getAllKeys();
|
||||
|
||||
const webhookKey = allKeys.find((key) => key.includes(path) && key.startsWith(httpMethod));
|
||||
|
||||
if (!webhookKey) return;
|
||||
|
||||
const { workflow } = this.registrations[webhookKey];
|
||||
const registration = await this.registrations.get(webhookKey);
|
||||
|
||||
if (!registration) return;
|
||||
|
||||
const { workflowEntity } = registration;
|
||||
|
||||
const workflow = this.toWorkflow(workflowEntity);
|
||||
|
||||
const webhookNode = Object.values(workflow.nodes).find(
|
||||
({ type, parameters, typeVersion }) =>
|
||||
parameters?.path === path &&
|
||||
|
|
@ -185,14 +186,13 @@ export class TestWebhooks implements IWebhookManager {
|
|||
*/
|
||||
async needsWebhook(
|
||||
workflowEntity: IWorkflowDb,
|
||||
workflow: Workflow,
|
||||
additionalData: IWorkflowExecuteAdditionalData,
|
||||
executionMode: WorkflowExecuteMode,
|
||||
activationMode: WorkflowActivateMode,
|
||||
sessionId?: string,
|
||||
destinationNode?: string,
|
||||
) {
|
||||
if (!workflow.id) throw new WorkflowMissingIdError(workflow);
|
||||
if (!workflowEntity.id) throw new WorkflowMissingIdError(workflowEntity);
|
||||
|
||||
const workflow = this.toWorkflow(workflowEntity);
|
||||
|
||||
const webhooks = WebhookHelpers.getWorkflowWebhooks(
|
||||
workflow,
|
||||
|
|
@ -205,37 +205,43 @@ export class TestWebhooks implements IWebhookManager {
|
|||
return false; // no webhooks found to start a workflow
|
||||
}
|
||||
|
||||
// 1+ webhook(s) required, so activate webhook(s)
|
||||
|
||||
const timeout = setTimeout(() => this.cancelWebhook(workflow.id), 2 * TIME.MINUTE);
|
||||
|
||||
const activatedKeys: string[] = [];
|
||||
const timeout = setTimeout(async () => this.cancelWebhook(workflow.id), 2 * TIME.MINUTE);
|
||||
|
||||
for (const webhook of webhooks) {
|
||||
const key = this.toWebhookKey(webhook);
|
||||
const key = this.registrations.toKey(webhook);
|
||||
const registration = await this.registrations.get(key);
|
||||
|
||||
if (this.registrations[key] && !webhook.webhookId) {
|
||||
if (registration && !webhook.webhookId) {
|
||||
throw new WebhookPathTakenError(webhook.node);
|
||||
}
|
||||
|
||||
activatedKeys.push(key);
|
||||
webhook.path = removeTrailingSlash(webhook.path);
|
||||
webhook.isTest = true;
|
||||
|
||||
this.setRegistration({
|
||||
sessionId,
|
||||
timeout,
|
||||
workflow,
|
||||
workflowEntity,
|
||||
destinationNode,
|
||||
webhook,
|
||||
});
|
||||
/**
|
||||
* Remove additional data from webhook because:
|
||||
*
|
||||
* - It is not needed for the test webhook to be executed.
|
||||
* - It contains circular refs that cannot be cached.
|
||||
*/
|
||||
const { workflowExecuteAdditionalData: _, ...rest } = webhook;
|
||||
|
||||
try {
|
||||
await this.activateWebhook(workflow, webhook, executionMode, activationMode);
|
||||
} catch (error) {
|
||||
activatedKeys.forEach((k) => delete this.registrations[k]);
|
||||
await workflow.createWebhookIfNotExists(webhook, NodeExecuteFunctions, 'manual', 'manual');
|
||||
|
||||
await this.registrations.register({
|
||||
sessionId,
|
||||
workflowEntity,
|
||||
destinationNode,
|
||||
webhook: rest as IWebhookData,
|
||||
});
|
||||
|
||||
this.timeouts[key] = timeout;
|
||||
} catch (error) {
|
||||
await this.deactivateWebhooks(workflow);
|
||||
|
||||
delete this.timeouts[key];
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -243,11 +249,21 @@ export class TestWebhooks implements IWebhookManager {
|
|||
return true;
|
||||
}
|
||||
|
||||
cancelWebhook(workflowId: string) {
|
||||
async cancelWebhook(workflowId: string) {
|
||||
let foundWebhook = false;
|
||||
|
||||
for (const key of Object.keys(this.registrations)) {
|
||||
const { sessionId, timeout, workflow, workflowEntity } = this.registrations[key];
|
||||
const allWebhookKeys = await this.registrations.getAllKeys();
|
||||
|
||||
for (const key of allWebhookKeys) {
|
||||
const registration = await this.registrations.get(key);
|
||||
|
||||
if (!registration) continue;
|
||||
|
||||
const { sessionId, workflowEntity } = registration;
|
||||
|
||||
const timeout = this.timeouts[key];
|
||||
|
||||
const workflow = this.toWorkflow(workflowEntity);
|
||||
|
||||
if (workflowEntity.id !== workflowId) continue;
|
||||
|
||||
|
|
@ -261,8 +277,6 @@ export class TestWebhooks implements IWebhookManager {
|
|||
}
|
||||
}
|
||||
|
||||
delete this.registrations[key];
|
||||
|
||||
if (!foundWebhook) {
|
||||
// As it removes all webhooks of the workflow execute only once
|
||||
void this.deactivateWebhooks(workflow);
|
||||
|
|
@ -274,45 +288,19 @@ export class TestWebhooks implements IWebhookManager {
|
|||
return foundWebhook;
|
||||
}
|
||||
|
||||
async activateWebhook(
|
||||
workflow: Workflow,
|
||||
webhook: IWebhookData,
|
||||
executionMode: WorkflowExecuteMode,
|
||||
activationMode: WorkflowActivateMode,
|
||||
) {
|
||||
webhook.path = removeTrailingSlash(webhook.path);
|
||||
|
||||
const key = this.toWebhookKey(webhook);
|
||||
|
||||
webhook.isTest = true;
|
||||
this.registrations[key].webhook = webhook;
|
||||
|
||||
try {
|
||||
await workflow.createWebhookIfNotExists(
|
||||
webhook,
|
||||
NodeExecuteFunctions,
|
||||
executionMode,
|
||||
activationMode,
|
||||
);
|
||||
} catch (error) {
|
||||
if (this.registrations[key]) delete this.registrations[key];
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
getActiveWebhook(httpMethod: IHttpRequestMethods, path: string, webhookId?: string) {
|
||||
const key = this.toWebhookKey({ httpMethod, path, webhookId });
|
||||
if (this.registrations[key] === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
async getActiveWebhook(httpMethod: IHttpRequestMethods, path: string, webhookId?: string) {
|
||||
const key = this.registrations.toKey({ httpMethod, path, webhookId });
|
||||
|
||||
let webhook: IWebhookData | undefined;
|
||||
let maxMatches = 0;
|
||||
const pathElementsSet = new Set(path.split('/'));
|
||||
// check if static elements match in path
|
||||
// if more results have been returned choose the one with the most static-route matches
|
||||
const dynamicWebhook = this.registrations[key].webhook;
|
||||
const registration = await this.registrations.get(key);
|
||||
|
||||
if (!registration) return;
|
||||
|
||||
const { webhook: dynamicWebhook } = registration;
|
||||
|
||||
const staticElements = dynamicWebhook.path.split('/').filter((ele) => !ele.startsWith(':'));
|
||||
const allStaticExist = staticElements.every((staticEle) => pathElementsSet.has(staticEle));
|
||||
|
|
@ -329,48 +317,47 @@ export class TestWebhooks implements IWebhookManager {
|
|||
return webhook;
|
||||
}
|
||||
|
||||
private toWebhookKey(webhook: Pick<IWebhookData, 'webhookId' | 'httpMethod' | 'path'>) {
|
||||
const { webhookId, httpMethod, path: webhookPath } = webhook;
|
||||
|
||||
if (!webhookId) return `${httpMethod}|${webhookPath}`;
|
||||
|
||||
let path = webhookPath;
|
||||
|
||||
if (path.startsWith(webhookId)) {
|
||||
const cutFromIndex = path.indexOf('/') + 1;
|
||||
|
||||
path = path.slice(cutFromIndex);
|
||||
}
|
||||
|
||||
return `${httpMethod}|${webhookId}|${path.split('/').length}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deactivate all registered webhooks of a workflow.
|
||||
* Deactivate all registered test webhooks of a workflow.
|
||||
*/
|
||||
async deactivateWebhooks(workflow: Workflow) {
|
||||
const webhooks = this.webhooksByWorkflow[workflow.id];
|
||||
const allRegistrations = await this.registrations.getAllRegistrations();
|
||||
|
||||
if (!webhooks) return false; // nothing to deactivate
|
||||
type WebhooksByWorkflow = { [workflowId: string]: IWebhookData[] };
|
||||
|
||||
const webhooksByWorkflow = allRegistrations.reduce<WebhooksByWorkflow>((acc, cur) => {
|
||||
const { workflowId } = cur.webhook;
|
||||
|
||||
acc[workflowId] ||= [];
|
||||
acc[workflowId].push(cur.webhook);
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const webhooks = webhooksByWorkflow[workflow.id];
|
||||
|
||||
if (!webhooks) return; // nothing to deactivate
|
||||
|
||||
for (const webhook of webhooks) {
|
||||
await workflow.deleteWebhook(webhook, NodeExecuteFunctions, 'internal', 'update');
|
||||
|
||||
const key = this.toWebhookKey(webhook);
|
||||
|
||||
delete this.registrations[key];
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
clearRegistrations() {
|
||||
this.registrations = {};
|
||||
}
|
||||
|
||||
setRegistration(registration: WebhookRegistration) {
|
||||
const key = this.toWebhookKey(registration.webhook);
|
||||
|
||||
this.registrations[key] = registration;
|
||||
await this.registrations.deregister(webhook);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `WorkflowEntity` from `typeorm` to a `Workflow` from `n8n-workflow`.
|
||||
*/
|
||||
private toWorkflow(workflowEntity: IWorkflowDb) {
|
||||
return new Workflow({
|
||||
id: workflowEntity.id,
|
||||
name: workflowEntity.name,
|
||||
nodes: workflowEntity.nodes,
|
||||
connections: workflowEntity.connections,
|
||||
active: false,
|
||||
nodeTypes: this.nodeTypes,
|
||||
staticData: undefined,
|
||||
settings: workflowEntity.settings,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import type { IWorkflowDb } from '@/Interfaces';
|
||||
import type { Workflow } from 'n8n-workflow';
|
||||
import { ApplicationError } from 'n8n-workflow';
|
||||
|
||||
export class WorkflowMissingIdError extends ApplicationError {
|
||||
constructor(workflow: Workflow) {
|
||||
constructor(workflow: Workflow | IWorkflowDb) {
|
||||
super('Detected ID-less worklfow', { extra: { workflow } });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ export class CacheService extends EventEmitter {
|
|||
return (this.cache as RedisCache)?.store?.isCacheable !== undefined;
|
||||
}
|
||||
|
||||
isMemoryCache(): boolean {
|
||||
return !this.isRedisCache();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the cache service.
|
||||
*
|
||||
|
|
@ -118,15 +122,15 @@ export class CacheService extends EventEmitter {
|
|||
* @param options.refreshTtl Optional ttl for the refreshFunction's set call
|
||||
* @param options.fallbackValue Optional value returned is cache is not hit and refreshFunction is not provided
|
||||
*/
|
||||
async getMany(
|
||||
async getMany<T = unknown[]>(
|
||||
keys: string[],
|
||||
options: {
|
||||
fallbackValues?: unknown[];
|
||||
refreshFunctionEach?: (key: string) => Promise<unknown>;
|
||||
refreshFunctionMany?: (keys: string[]) => Promise<unknown[]>;
|
||||
fallbackValues?: T[];
|
||||
refreshFunctionEach?: (key: string) => Promise<T>;
|
||||
refreshFunctionMany?: (keys: string[]) => Promise<T[]>;
|
||||
refreshTtl?: number;
|
||||
} = {},
|
||||
): Promise<unknown[]> {
|
||||
): Promise<T[]> {
|
||||
if (keys.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
|
@ -136,7 +140,7 @@ export class CacheService extends EventEmitter {
|
|||
}
|
||||
if (!values.includes(undefined)) {
|
||||
this.emit(this.metricsCounterEvents.cacheHit);
|
||||
return values;
|
||||
return values as T[];
|
||||
}
|
||||
this.emit(this.metricsCounterEvents.cacheMiss);
|
||||
if (options.refreshFunctionEach) {
|
||||
|
|
@ -155,7 +159,7 @@ export class CacheService extends EventEmitter {
|
|||
values[i] = refreshValue;
|
||||
}
|
||||
}
|
||||
return values;
|
||||
return values as T[];
|
||||
}
|
||||
if (options.refreshFunctionMany) {
|
||||
this.emit(this.metricsCounterEvents.cacheUpdate);
|
||||
|
|
@ -170,9 +174,9 @@ export class CacheService extends EventEmitter {
|
|||
newKV.push([keys[i], refreshValues[i]]);
|
||||
}
|
||||
await this.setMany(newKV, options.refreshTtl);
|
||||
return refreshValues;
|
||||
return refreshValues as T[];
|
||||
}
|
||||
return options.fallbackValues ?? values;
|
||||
return (options.fallbackValues ?? values) as T[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -196,6 +200,7 @@ export class CacheService extends EventEmitter {
|
|||
throw new ApplicationError('Value is not cacheable');
|
||||
}
|
||||
}
|
||||
|
||||
await this.cache?.store.set(key, value, ttl);
|
||||
}
|
||||
|
||||
|
|
@ -284,7 +289,9 @@ export class CacheService extends EventEmitter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Return all keys in the cache.
|
||||
* Return all keys in the cache. Not recommended for production use.
|
||||
*
|
||||
* https://redis.io/commands/keys/
|
||||
*/
|
||||
async keys(): Promise<string[]> {
|
||||
return this.cache?.store.keys() ?? [];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
import { Service } from 'typedi';
|
||||
import { CacheService } from './cache.service';
|
||||
import { ApplicationError, type IWebhookData } from 'n8n-workflow';
|
||||
import type { IWorkflowDb } from '@/Interfaces';
|
||||
|
||||
export type TestWebhookRegistration = {
|
||||
sessionId?: string;
|
||||
workflowEntity: IWorkflowDb;
|
||||
destinationNode?: string;
|
||||
webhook: IWebhookData;
|
||||
};
|
||||
|
||||
@Service()
|
||||
export class TestWebhookRegistrationsService {
|
||||
constructor(private readonly cacheService: CacheService) {}
|
||||
|
||||
private readonly cacheKey = 'test-webhook';
|
||||
|
||||
async register(registration: TestWebhookRegistration) {
|
||||
const key = this.toKey(registration.webhook);
|
||||
|
||||
await this.cacheService.set(key, registration);
|
||||
}
|
||||
|
||||
async deregister(arg: IWebhookData | string) {
|
||||
if (typeof arg === 'string') {
|
||||
await this.cacheService.delete(arg);
|
||||
} else {
|
||||
const key = this.toKey(arg);
|
||||
await this.cacheService.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
async get(key: string) {
|
||||
return this.cacheService.get<TestWebhookRegistration>(key);
|
||||
}
|
||||
|
||||
async getAllKeys() {
|
||||
const keys = await this.cacheService.keys();
|
||||
|
||||
if (this.cacheService.isMemoryCache()) {
|
||||
return keys.filter((key) => key.startsWith(this.cacheKey));
|
||||
}
|
||||
|
||||
const prefix = 'n8n:cache'; // prepended by Redis cache
|
||||
const extendedCacheKey = `${prefix}:${this.cacheKey}`;
|
||||
|
||||
return keys
|
||||
.filter((key) => key.startsWith(extendedCacheKey))
|
||||
.map((key) => key.slice(`${prefix}:`.length));
|
||||
}
|
||||
|
||||
async getAllRegistrations() {
|
||||
const keys = await this.getAllKeys();
|
||||
|
||||
return this.cacheService.getMany<TestWebhookRegistration>(keys);
|
||||
}
|
||||
|
||||
async updateWebhookProperties(newProperties: IWebhookData) {
|
||||
const key = this.toKey(newProperties);
|
||||
|
||||
const registration = await this.cacheService.get<TestWebhookRegistration>(key);
|
||||
|
||||
if (!registration) {
|
||||
throw new ApplicationError('Failed to find test webhook registration', { extra: { key } });
|
||||
}
|
||||
|
||||
registration.webhook = newProperties;
|
||||
|
||||
await this.cacheService.set(key, registration);
|
||||
}
|
||||
|
||||
async deregisterAll() {
|
||||
const testWebhookKeys = await this.getAllKeys();
|
||||
|
||||
await this.cacheService.deleteMany(testWebhookKeys);
|
||||
}
|
||||
|
||||
toKey(webhook: Pick<IWebhookData, 'webhookId' | 'httpMethod' | 'path'>) {
|
||||
const { webhookId, httpMethod, path: webhookPath } = webhook;
|
||||
|
||||
if (!webhookId) return `${this.cacheKey}:${httpMethod}|${webhookPath}`;
|
||||
|
||||
let path = webhookPath;
|
||||
|
||||
if (path.startsWith(webhookId)) {
|
||||
const cutFromIndex = path.indexOf('/') + 1;
|
||||
|
||||
path = path.slice(cutFromIndex);
|
||||
}
|
||||
|
||||
return `${this.cacheKey}:${httpMethod}|${webhookId}|${path.split('/').length}`;
|
||||
}
|
||||
}
|
||||
|
|
@ -302,9 +302,6 @@ export class WorkflowService {
|
|||
user: User,
|
||||
sessionId?: string,
|
||||
) {
|
||||
const EXECUTION_MODE = 'manual';
|
||||
const ACTIVATION_MODE = 'manual';
|
||||
|
||||
const pinnedTrigger = this.findPinnedTrigger(workflowData, startNodes, pinData);
|
||||
|
||||
// If webhooks nodes exist and are active we have to wait for till we receive a call
|
||||
|
|
@ -315,25 +312,11 @@ export class WorkflowService {
|
|||
startNodes.length === 0 ||
|
||||
destinationNode === undefined)
|
||||
) {
|
||||
const workflow = new Workflow({
|
||||
id: workflowData.id?.toString(),
|
||||
name: workflowData.name,
|
||||
nodes: workflowData.nodes,
|
||||
connections: workflowData.connections,
|
||||
active: false,
|
||||
nodeTypes: this.nodeTypes,
|
||||
staticData: undefined,
|
||||
settings: workflowData.settings,
|
||||
});
|
||||
|
||||
const additionalData = await WorkflowExecuteAdditionalData.getBase(user.id);
|
||||
|
||||
const needsWebhook = await this.testWebhooks.needsWebhook(
|
||||
workflowData,
|
||||
workflow,
|
||||
additionalData,
|
||||
EXECUTION_MODE,
|
||||
ACTIVATION_MODE,
|
||||
sessionId,
|
||||
destinationNode,
|
||||
);
|
||||
|
|
@ -347,7 +330,7 @@ export class WorkflowService {
|
|||
// Start the workflow
|
||||
const data: IWorkflowExecutionDataProcess = {
|
||||
destinationNode,
|
||||
executionMode: EXECUTION_MODE,
|
||||
executionMode: 'manual',
|
||||
runData,
|
||||
pinData,
|
||||
sessionId,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { generateNanoId } from '@/databases/utils/generators';
|
|||
import { NotFoundError } from '@/errors/response-errors/not-found.error';
|
||||
import * as WebhookHelpers from '@/WebhookHelpers';
|
||||
|
||||
import type { IWorkflowDb, WebhookRegistration, WebhookRequest } from '@/Interfaces';
|
||||
import type { IWorkflowDb, WebhookRequest } from '@/Interfaces';
|
||||
import type {
|
||||
IWebhookData,
|
||||
IWorkflowExecuteAdditionalData,
|
||||
|
|
@ -14,18 +14,19 @@ import type {
|
|||
WorkflowActivateMode,
|
||||
WorkflowExecuteMode,
|
||||
} from 'n8n-workflow';
|
||||
import type {
|
||||
TestWebhookRegistrationsService,
|
||||
TestWebhookRegistration,
|
||||
} from '@/services/test-webhook-registrations.service';
|
||||
|
||||
describe('TestWebhooks', () => {
|
||||
const testWebhooks = new TestWebhooks(mock(), mock());
|
||||
const registrations = mock<TestWebhookRegistrationsService>();
|
||||
const testWebhooks = new TestWebhooks(mock(), mock(), registrations);
|
||||
|
||||
beforeAll(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testWebhooks.clearRegistrations();
|
||||
});
|
||||
|
||||
const httpMethod = 'GET';
|
||||
const path = uuid();
|
||||
const workflowId = generateNanoId();
|
||||
|
|
@ -39,7 +40,6 @@ describe('TestWebhooks', () => {
|
|||
describe('needsWebhook()', () => {
|
||||
type NeedsWebhookArgs = [
|
||||
IWorkflowDb,
|
||||
Workflow,
|
||||
IWorkflowExecuteAdditionalData,
|
||||
WorkflowExecuteMode,
|
||||
WorkflowActivateMode,
|
||||
|
|
@ -48,37 +48,33 @@ describe('TestWebhooks', () => {
|
|||
const workflow = mock<Workflow>({ id: workflowId });
|
||||
|
||||
const args: NeedsWebhookArgs = [
|
||||
mock<IWorkflowDb>({ id: workflowId }),
|
||||
workflow,
|
||||
mock<IWorkflowDb>({ id: workflowId, nodes: [] }),
|
||||
mock<IWorkflowExecuteAdditionalData>(),
|
||||
'manual',
|
||||
'manual',
|
||||
];
|
||||
|
||||
test('should return true and activate webhook if needed', async () => {
|
||||
test('if webhook is needed, should return true and activate webhook', async () => {
|
||||
jest.spyOn(WebhookHelpers, 'getWorkflowWebhooks').mockReturnValue([webhook]);
|
||||
const activateWebhookSpy = jest.spyOn(testWebhooks, 'activateWebhook');
|
||||
|
||||
const needsWebhook = await testWebhooks.needsWebhook(...args);
|
||||
|
||||
expect(needsWebhook).toBe(true);
|
||||
expect(activateWebhookSpy).toHaveBeenCalledWith(workflow, webhook, 'manual', 'manual');
|
||||
});
|
||||
|
||||
test('should deactivate webhooks on failure to activate', async () => {
|
||||
test('if webhook activation fails, should deactivate workflow webhooks', async () => {
|
||||
const msg = 'Failed to add webhook to active webhooks';
|
||||
|
||||
jest.spyOn(WebhookHelpers, 'getWorkflowWebhooks').mockReturnValue([webhook]);
|
||||
jest.spyOn(testWebhooks, 'activateWebhook').mockRejectedValue(new Error(msg));
|
||||
const deactivateWebhooksSpy = jest.spyOn(testWebhooks, 'deactivateWebhooks');
|
||||
jest.spyOn(registrations, 'register').mockRejectedValueOnce(new Error(msg));
|
||||
registrations.getAllRegistrations.mockResolvedValue([]);
|
||||
|
||||
const needsWebhook = testWebhooks.needsWebhook(...args);
|
||||
|
||||
await expect(needsWebhook).rejects.toThrowError(msg);
|
||||
expect(deactivateWebhooksSpy).toHaveBeenCalledWith(workflow);
|
||||
});
|
||||
|
||||
test('should return false if no webhook to start workflow', async () => {
|
||||
test('if no webhook is found to start workflow, should return false', async () => {
|
||||
webhook.webhookDescription.restartWebhook = true;
|
||||
jest.spyOn(WebhookHelpers, 'getWorkflowWebhooks').mockReturnValue([webhook]);
|
||||
|
||||
|
|
@ -89,8 +85,8 @@ describe('TestWebhooks', () => {
|
|||
});
|
||||
|
||||
describe('executeWebhook()', () => {
|
||||
test('should throw if webhook is not registered', async () => {
|
||||
jest.spyOn(testWebhooks, 'getActiveWebhook').mockReturnValue(webhook);
|
||||
test('if webhook is not registered, should throw', async () => {
|
||||
jest.spyOn(testWebhooks, 'getActiveWebhook').mockResolvedValue(webhook);
|
||||
jest.spyOn(testWebhooks, 'getWebhookMethods').mockResolvedValue([]);
|
||||
|
||||
const promise = testWebhooks.executeWebhook(
|
||||
|
|
@ -101,18 +97,16 @@ describe('TestWebhooks', () => {
|
|||
await expect(promise).rejects.toThrowError(WebhookNotFoundError);
|
||||
});
|
||||
|
||||
test('should throw if webhook node is registered but missing from workflow', async () => {
|
||||
jest.spyOn(testWebhooks, 'getActiveWebhook').mockReturnValue(webhook);
|
||||
test('if webhook is registered but missing from workflow, should throw', async () => {
|
||||
jest.spyOn(testWebhooks, 'getActiveWebhook').mockResolvedValue(webhook);
|
||||
jest.spyOn(testWebhooks, 'getWebhookMethods').mockResolvedValue([]);
|
||||
|
||||
const registration = mock<WebhookRegistration>({
|
||||
const registration = mock<TestWebhookRegistration>({
|
||||
sessionId: 'some-session-id',
|
||||
timeout: mock<NodeJS.Timeout>(),
|
||||
workflowEntity: mock<IWorkflowDb>({}),
|
||||
workflow: mock<Workflow>(),
|
||||
});
|
||||
|
||||
testWebhooks.setRegistration(registration);
|
||||
await registrations.register(registration);
|
||||
|
||||
const promise = testWebhooks.executeWebhook(
|
||||
mock<WebhookRequest>({ params: { path } }),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
import type { CacheService } from '@/services/cache.service';
|
||||
import type { TestWebhookRegistration } from '@/services/test-webhook-registrations.service';
|
||||
import { TestWebhookRegistrationsService } from '@/services/test-webhook-registrations.service';
|
||||
import { mock } from 'jest-mock-extended';
|
||||
|
||||
describe('TestWebhookRegistrationsService', () => {
|
||||
const cacheService = mock<CacheService>();
|
||||
const registrations = new TestWebhookRegistrationsService(cacheService);
|
||||
|
||||
const registration = mock<TestWebhookRegistration>({
|
||||
webhook: { httpMethod: 'GET', path: 'hello', webhookId: undefined },
|
||||
});
|
||||
|
||||
const key = 'test-webhook:GET|hello';
|
||||
const fullCacheKey = `n8n:cache:${key}`;
|
||||
|
||||
describe('register()', () => {
|
||||
test('should register a test webhook registration', async () => {
|
||||
await registrations.register(registration);
|
||||
|
||||
expect(cacheService.set).toHaveBeenCalledWith(key, registration);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deregister()', () => {
|
||||
test('should deregister a test webhook registration', async () => {
|
||||
await registrations.register(registration);
|
||||
|
||||
await registrations.deregister(key);
|
||||
|
||||
expect(cacheService.delete).toHaveBeenCalledWith(key);
|
||||
});
|
||||
});
|
||||
|
||||
describe('get()', () => {
|
||||
test('should retrieve a test webhook registration', async () => {
|
||||
cacheService.get.mockResolvedValueOnce(registration);
|
||||
|
||||
const promise = registrations.get(key);
|
||||
|
||||
await expect(promise).resolves.toBe(registration);
|
||||
});
|
||||
|
||||
test('should return undefined if no such test webhook registration was found', async () => {
|
||||
cacheService.get.mockResolvedValueOnce(undefined);
|
||||
|
||||
const promise = registrations.get(key);
|
||||
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAllKeys()', () => {
|
||||
test('should retrieve all test webhook registration keys', async () => {
|
||||
cacheService.keys.mockResolvedValueOnce([fullCacheKey]);
|
||||
|
||||
const result = await registrations.getAllKeys();
|
||||
|
||||
expect(result).toEqual([key]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAllRegistrations()', () => {
|
||||
test('should retrieve all test webhook registrations', async () => {
|
||||
cacheService.keys.mockResolvedValueOnce([fullCacheKey]);
|
||||
cacheService.getMany.mockResolvedValueOnce([registration]);
|
||||
|
||||
const result = await registrations.getAllRegistrations();
|
||||
|
||||
expect(result).toEqual([registration]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateWebhookProperties()', () => {
|
||||
test('should update the properties of a test webhook registration', async () => {
|
||||
cacheService.get.mockResolvedValueOnce(registration);
|
||||
|
||||
const newProperties = { ...registration.webhook, isTest: true };
|
||||
|
||||
await registrations.updateWebhookProperties(newProperties);
|
||||
|
||||
registration.webhook = newProperties;
|
||||
|
||||
expect(cacheService.set).toHaveBeenCalledWith(key, registration);
|
||||
|
||||
delete registration.webhook.isTest;
|
||||
});
|
||||
});
|
||||
|
||||
describe('deregisterAll()', () => {
|
||||
test('should deregister all test webhook registrations', async () => {
|
||||
cacheService.keys.mockResolvedValueOnce([fullCacheKey]);
|
||||
|
||||
await registrations.deregisterAll();
|
||||
|
||||
expect(cacheService.delete).toHaveBeenCalledWith(key);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toKey()', () => {
|
||||
test('should convert a test webhook registration to a key', () => {
|
||||
const result = registrations.toKey(registration.webhook);
|
||||
|
||||
expect(result).toBe(key);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue