diff --git a/packages/cli/package.json b/packages/cli/package.json index 03229331c5..1b4390d61e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -136,6 +136,7 @@ "dotenv": "^8.0.0", "express": "^4.16.4", "express-openapi-validator": "^4.13.6", + "express-prom-bundle": "^6.6.0", "fast-glob": "^3.2.5", "flatted": "^3.2.4", "google-timezones-json": "^1.0.2", diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts index 6530f562fd..32fe4e8dbf 100644 --- a/packages/cli/src/InternalHooks.ts +++ b/packages/cli/src/InternalHooks.ts @@ -182,6 +182,7 @@ export class InternalHooksClass implements IInternalHooksClass { workflow: IWorkflowBase, nodeName: string, ): Promise { + const nodeInWorkflow = workflow.nodes.find((node) => node.name === nodeName); void eventBus.sendNodeEvent({ eventName: 'n8n.node.started', payload: { @@ -189,6 +190,7 @@ export class InternalHooksClass implements IInternalHooksClass { nodeName, workflowId: workflow.id?.toString(), workflowName: workflow.name, + nodeType: nodeInWorkflow?.type, }, }); } @@ -198,6 +200,7 @@ export class InternalHooksClass implements IInternalHooksClass { workflow: IWorkflowBase, nodeName: string, ): Promise { + const nodeInWorkflow = workflow.nodes.find((node) => node.name === nodeName); void eventBus.sendNodeEvent({ eventName: 'n8n.node.finished', payload: { @@ -205,6 +208,7 @@ export class InternalHooksClass implements IInternalHooksClass { nodeName, workflowId: workflow.id?.toString(), workflowName: workflow.name, + nodeType: nodeInWorkflow?.type, }, }); } diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 02278f08a7..5d47fbeefd 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -73,7 +73,6 @@ import jwt from 'jsonwebtoken'; import jwks from 'jwks-rsa'; // @ts-ignore import timezones from 'google-timezones-json'; -import promClient, { Registry } from 'prom-client'; import history from 'connect-history-api-fallback'; import config from '@/config'; @@ -154,6 +153,7 @@ import { licenseController } from './license/license.controller'; import { corsMiddleware } from './middlewares/cors'; import { initEvents } from './events'; import { AbstractServer } from './AbstractServer'; +import { configureMetrics } from './metrics'; const exec = promisify(callbackExec); @@ -321,15 +321,7 @@ class Server extends AbstractServer { } async configure(): Promise { - const enableMetrics = config.getEnv('endpoints.metrics.enable'); - let register: Registry; - - if (enableMetrics) { - const prefix = config.getEnv('endpoints.metrics.prefix'); - register = new promClient.Registry(); - register.setDefaultLabels({ prefix }); - promClient.collectDefaultMetrics({ register }); - } + configureMetrics(this.app); this.frontendSettings.isNpmAvailable = await exec('npm --version') .then(() => true) @@ -590,17 +582,6 @@ class Server extends AbstractServer { this.app.use(`/${this.restEndpoint}/nodes`, nodesController); } - // ---------------------------------------- - // Metrics - // ---------------------------------------- - if (enableMetrics) { - this.app.get('/metrics', async (req: express.Request, res: express.Response) => { - const response = await register.metrics(); - res.setHeader('Content-Type', register.contentType); - ResponseHelper.sendSuccessResponse(res, response, true, 200); - }); - } - // ---------------------------------------- // Workflow // ---------------------------------------- diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index ef55c7d64a..4ea7765fd5 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -573,7 +573,7 @@ export const schema = { format: 'Boolean', default: false, env: 'N8N_METRICS', - doc: 'Enable metrics endpoint', + doc: 'Enable /metrics endpoint. Default: false', }, prefix: { format: String, @@ -581,6 +581,54 @@ export const schema = { env: 'N8N_METRICS_PREFIX', doc: 'An optional prefix for metric names. Default: n8n_', }, + includeDefaultMetrics: { + format: Boolean, + default: true, + env: 'N8N_METRICS_INCLUDE_DEFAULT_METRICS', + doc: 'Whether to expose default system and node.js metrics. Default: true', + }, + includeWorkflowIdLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL', + doc: 'Whether to include a label for the workflow ID on workflow metrics. Default: false', + }, + includeNodeTypeLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_NODE_TYPE_LABEL', + doc: 'Whether to include a label for the node type on node metrics. Default: false', + }, + includeCredentialTypeLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_CREDENTIAL_TYPE_LABEL', + doc: 'Whether to include a label for the credential type on credential metrics. Default: false', + }, + includeApiEndpoints: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_API_ENDPOINTS', + doc: 'Whether to expose metrics for API endpoints. Default: false', + }, + includeApiPathLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_API_PATH_LABEL', + doc: 'Whether to include a label for the path of API invocations. Default: false', + }, + includeApiMethodLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_API_METHOD_LABEL', + doc: 'Whether to include a label for the HTTP method (GET, POST, ...) of API invocations. Default: false', + }, + includeApiStatusCodeLabel: { + format: Boolean, + default: false, + env: 'N8N_METRICS_INCLUDE_API_STATUS_CODE_LABEL', + doc: 'Whether to include a label for the HTTP status code (200, 404, ...) of API invocations. Default: false', + }, }, rest: { format: String, diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts index 5c59095f8c..baf8089964 100644 --- a/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts @@ -35,6 +35,11 @@ export interface EventPayloadAudit extends AbstractEventPayload { userEmail?: string; firstName?: string; lastName?: string; + credentialName?: string; + credentialType?: string; + credentialId?: string; + workflowId?: string; + workflowName?: string; } export interface EventMessageAuditOptions extends AbstractEventMessageOptions { diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts index a7c50f1211..cbf726d941 100644 --- a/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts @@ -11,6 +11,11 @@ export type EventNamesNodeType = typeof eventNamesNode[number]; // -------------------------------------- export interface EventPayloadNode extends AbstractEventPayload { msg?: string; + executionId: string; + nodeName: string; + workflowId?: string; + workflowName: string; + nodeType?: string; } export interface EventMessageNodeOptions extends AbstractEventMessageOptions { diff --git a/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts index ea773a3b34..2751dc0f91 100644 --- a/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts +++ b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts @@ -6,7 +6,10 @@ import { MessageEventBusLogWriter } from '../MessageEventBusWriter/MessageEventB import EventEmitter from 'events'; import config from '@/config'; import * as Db from '@/Db'; -import { messageEventBusDestinationFromDb } from '../MessageEventBusDestination/Helpers.ee'; +import { + messageEventBusDestinationFromDb, + incrementPrometheusMetric, +} from '../MessageEventBusDestination/Helpers.ee'; import uniqby from 'lodash.uniqby'; import { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm'; import { @@ -205,6 +208,10 @@ class MessageEventBus extends EventEmitter { } private async emitMessage(msg: EventMessageTypes) { + if (config.getEnv('endpoints.metrics.enable')) { + await incrementPrometheusMetric(msg); + } + // generic emit for external modules to capture events // this is for internal use ONLY and not for use with custom destinations! this.emit('message', msg); diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts index abc08a87e1..712b1170dc 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts @@ -1,6 +1,13 @@ /* eslint-disable import/no-cycle */ -import { MessageEventBusDestinationTypeNames } from 'n8n-workflow'; import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity'; +import { promClient } from '@/metrics'; +import { + EventMessageTypeNames, + LoggerProxy, + MessageEventBusDestinationTypeNames, +} from 'n8n-workflow'; +import config from '../../config'; +import type { EventMessageTypes } from '../EventMessageClasses'; import type { MessageEventBusDestination } from './MessageEventBusDestination.ee'; import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee'; import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee'; @@ -24,3 +31,85 @@ export function messageEventBusDestinationFromDb( } return null; } + +const prometheusCounters: Record | null> = {}; + +function getMetricNameForEvent(event: EventMessageTypes): string { + const prefix = config.getEnv('endpoints.metrics.prefix'); + return prefix + event.eventName.replace('n8n.', '').replace(/\./g, '_') + '_total'; +} + +function getLabelValueForNode(nodeType: string): string { + return nodeType.replace('n8n-nodes-', '').replace(/\./g, '_'); +} + +function getLabelValueForCredential(credentialType: string): string { + return credentialType.replace(/\./g, '_'); +} + +function getLabelsForEvent(event: EventMessageTypes): Record { + switch (event.__type) { + case EventMessageTypeNames.audit: + if (event.eventName.startsWith('n8n.audit.user.credentials')) { + return config.getEnv('endpoints.metrics.includeCredentialTypeLabel') + ? { + credential_type: getLabelValueForCredential( + event.payload.credentialType ?? 'unknown', + ), + } + : {}; + } + + if (event.eventName.startsWith('n8n.audit.workflow')) { + return config.getEnv('endpoints.metrics.includeWorkflowIdLabel') + ? { workflow_id: event.payload.workflowId?.toString() ?? 'unknown' } + : {}; + } + break; + + case EventMessageTypeNames.node: + return config.getEnv('endpoints.metrics.includeNodeTypeLabel') + ? { node_type: getLabelValueForNode(event.payload.nodeType ?? 'unknown') } + : {}; + + case EventMessageTypeNames.workflow: + return config.getEnv('endpoints.metrics.includeWorkflowIdLabel') + ? { workflow_id: event.payload.workflowId?.toString() ?? 'unknown' } + : {}; + } + + return {}; +} + +function getCounterSingletonForEvent(event: EventMessageTypes) { + if (!prometheusCounters[event.eventName]) { + const metricName = getMetricNameForEvent(event); + + if (!promClient.validateMetricName(metricName)) { + LoggerProxy.debug(`Invalid metric name: ${metricName}. Ignoring it!`); + prometheusCounters[event.eventName] = null; + return null; + } + + const counter = new promClient.Counter({ + name: metricName, + help: `Total number of ${event.eventName} events.`, + labelNames: Object.keys(getLabelsForEvent(event)), + }); + + promClient.register.registerMetric(counter); + prometheusCounters[event.eventName] = counter; + } + + return prometheusCounters[event.eventName]; +} + +export async function incrementPrometheusMetric(event: EventMessageTypes): Promise { + const counter = getCounterSingletonForEvent(event); + + if (!counter) { + return; + } + + counter.inc(getLabelsForEvent(event)); +} diff --git a/packages/cli/src/eventbus/eventBusRoutes.ts b/packages/cli/src/eventbus/eventBusRoutes.ts index d5a30fc9fb..ea1108d840 100644 --- a/packages/cli/src/eventbus/eventBusRoutes.ts +++ b/packages/cli/src/eventbus/eventBusRoutes.ts @@ -32,6 +32,7 @@ import { } from 'n8n-workflow'; import { User } from '../databases/entities/User'; import * as ResponseHelper from '@/ResponseHelper'; +import { EventMessageNode, EventMessageNodeOptions } from './EventMessageClasses/EventMessageNode'; export const eventBusRouter = express.Router(); @@ -116,6 +117,9 @@ eventBusRouter.post( case EventMessageTypeNames.audit: msg = new EventMessageAudit(req.body as EventMessageAuditOptions); break; + case EventMessageTypeNames.node: + msg = new EventMessageNode(req.body as EventMessageNodeOptions); + break; case EventMessageTypeNames.generic: default: msg = new EventMessageGeneric(req.body); diff --git a/packages/cli/src/metrics/index.ts b/packages/cli/src/metrics/index.ts new file mode 100644 index 0000000000..80b211494d --- /dev/null +++ b/packages/cli/src/metrics/index.ts @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/no-use-before-define */ +import config from '@/config'; +import { N8N_VERSION } from '@/constants'; +import * as ResponseHelper from '@/ResponseHelper'; +import express from 'express'; +import promBundle from 'express-prom-bundle'; +import promClient from 'prom-client'; +import semverParse from 'semver/functions/parse'; + +export { promClient }; + +export function configureMetrics(app: express.Application) { + if (!config.getEnv('endpoints.metrics.enable')) { + return; + } + + setupDefaultMetrics(); + setupN8nVersionMetric(); + setupApiMetrics(app); + mountMetricsEndpoint(app); +} + +function setupN8nVersionMetric() { + const n8nVersion = semverParse(N8N_VERSION || '0.0.0'); + + if (n8nVersion) { + const versionGauge = new promClient.Gauge({ + name: config.getEnv('endpoints.metrics.prefix') + 'version_info', + help: 'n8n version info.', + labelNames: ['version', 'major', 'minor', 'patch'], + }); + + versionGauge.set( + { + version: 'v' + n8nVersion.version, + major: n8nVersion.major, + minor: n8nVersion.minor, + patch: n8nVersion.patch, + }, + 1, + ); + } +} + +function setupDefaultMetrics() { + if (config.getEnv('endpoints.metrics.includeDefaultMetrics')) { + promClient.collectDefaultMetrics(); + } +} + +function setupApiMetrics(app: express.Application) { + if (config.getEnv('endpoints.metrics.includeApiEndpoints')) { + const metricsMiddleware = promBundle({ + autoregister: false, + includeUp: false, + includePath: config.getEnv('endpoints.metrics.includeApiPathLabel'), + includeMethod: config.getEnv('endpoints.metrics.includeApiMethodLabel'), + includeStatusCode: config.getEnv('endpoints.metrics.includeApiStatusCodeLabel'), + }); + + app.use(['/rest/', '/webhook/', 'webhook-test/', '/api/'], metricsMiddleware); + } +} + +function mountMetricsEndpoint(app: express.Application) { + app.get('/metrics', async (req: express.Request, res: express.Response) => { + const response = await promClient.register.metrics(); + res.setHeader('Content-Type', promClient.register.contentType); + ResponseHelper.sendSuccessResponse(res, response, true, 200); + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d148b4325..a78aa1428f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,6 +160,7 @@ importers: dotenv: ^8.0.0 express: ^4.16.4 express-openapi-validator: ^4.13.6 + express-prom-bundle: ^6.6.0 fast-glob: ^3.2.5 flatted: ^3.2.4 google-timezones-json: ^1.0.2 @@ -252,6 +253,7 @@ importers: dotenv: 8.6.0 express: 4.18.2 express-openapi-validator: 4.13.8 + express-prom-bundle: 6.6.0_prom-client@13.2.0 fast-glob: 3.2.12 flatted: 3.2.7 google-timezones-json: 1.0.2 @@ -11561,6 +11563,17 @@ packages: path-to-regexp: 6.2.1 dev: false + /express-prom-bundle/6.6.0_prom-client@13.2.0: + resolution: {integrity: sha512-tZh2P2p5a8/yxQ5VbRav011Poa4R0mHqdFwn9Swe/obXDe5F0jY9wtRAfNYnqk4LXY7akyvR/nrvAHxQPWUjsQ==} + engines: {node: '>=10'} + peerDependencies: + prom-client: '>=12.0.0' + dependencies: + on-finished: 2.4.1 + prom-client: 13.2.0 + url-value-parser: 2.2.0 + dev: false + /express/4.18.2: resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} engines: {node: '>= 0.10.0'} @@ -21274,6 +21287,11 @@ packages: querystringify: 2.2.0 requires-port: 1.0.0 + /url-value-parser/2.2.0: + resolution: {integrity: sha512-yIQdxJpgkPamPPAPuGdS7Q548rLhny42tg8d4vyTNzFqvOnwqrgHXvgehT09U7fwrzxi3RxCiXjoNUNnNOlQ8A==} + engines: {node: '>=6.0.0'} + dev: false + /url/0.10.3: resolution: {integrity: sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==} dependencies: