refactor(core): Clean up Prometheus service (no-changelog) (#10068)

This commit is contained in:
Iván Ovejero 2024-07-18 10:27:35 +02:00 committed by GitHub
parent 14b12f844d
commit 2c710ac7d2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 311 additions and 224 deletions

View file

@ -168,7 +168,7 @@ export class Server extends AbstractServer {
async configure(): Promise<void> {
if (config.getEnv('endpoints.metrics.enable')) {
const { PrometheusMetricsService } = await import('@/metrics/prometheus-metrics.service');
await Container.get(PrometheusMetricsService).configureMetrics(this.app);
await Container.get(PrometheusMetricsService).init(this.app);
}
const { frontendService } = this;

View file

@ -309,7 +309,7 @@ export class MessageEventBus extends EventEmitter {
}
private async emitMessage(msg: EventMessageTypes) {
this.emit('metrics.messageEventBus.Event', msg);
this.emit('metrics.eventBus.event', msg);
// generic emit for external modules to capture events
// this is for internal use ONLY and not for use with custom destinations!

View file

@ -20,11 +20,11 @@ describe('PrometheusMetricsService', () => {
config.load(config.default);
});
describe('configureMetrics', () => {
describe('init', () => {
it('should set up `n8n_version_info`', async () => {
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(promClient.Gauge).toHaveBeenCalledWith({
name: 'n8n_version_info',
@ -34,52 +34,55 @@ describe('PrometheusMetricsService', () => {
});
it('should set up default metrics collection with `prom-client`', async () => {
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(promClient.collectDefaultMetrics).toHaveBeenCalled();
});
it('should set up `n8n_cache_hits_total`', async () => {
config.set('endpoints.metrics.includeCacheMetrics', true);
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(promClient.Counter).toHaveBeenCalledWith({
name: 'n8n_cache_hits_total',
help: 'Total number of cache hits.',
labelNames: ['cache'],
});
// @ts-expect-error private field
expect(service.counters.cacheHitsTotal?.inc).toHaveBeenCalledWith(0);
});
it('should set up `n8n_cache_misses_total`', async () => {
config.set('endpoints.metrics.includeCacheMetrics', true);
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(promClient.Counter).toHaveBeenCalledWith({
name: 'n8n_cache_misses_total',
help: 'Total number of cache misses.',
labelNames: ['cache'],
});
// @ts-expect-error private field
expect(service.counters.cacheMissesTotal?.inc).toHaveBeenCalledWith(0);
});
it('should set up `n8n_cache_updates_total`', async () => {
config.set('endpoints.metrics.includeCacheMetrics', true);
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(promClient.Counter).toHaveBeenCalledWith({
name: 'n8n_cache_updates_total',
help: 'Total number of cache updates.',
labelNames: ['cache'],
});
// @ts-expect-error private field
expect(service.counters.cacheUpdatesTotal?.inc).toHaveBeenCalledWith(0);
});
@ -88,11 +91,11 @@ describe('PrometheusMetricsService', () => {
config.set('endpoints.metrics.includeApiPathLabel', true);
config.set('endpoints.metrics.includeApiMethodLabel', true);
config.set('endpoints.metrics.includeApiStatusCodeLabel', true);
const service = new PrometheusMetricsService(mock(), mock(), mock());
const service = new PrometheusMetricsService(mock(), mock());
const app = mock<express.Application>();
await service.configureMetrics(app);
await service.init(app);
expect(promBundle).toHaveBeenCalledWith({
autoregister: false,
@ -103,21 +106,18 @@ describe('PrometheusMetricsService', () => {
});
expect(app.use).toHaveBeenCalledWith(
['/rest/', '/webhook/', 'webhook-test/', '/api/'],
['/rest/', '/webhook/', '/webhook-waiting/', '/form-waiting/', '/webhook-test/', '/api/'],
expect.any(Function),
);
});
it('should set up event bus metrics', async () => {
const eventBus = mock<MessageEventBus>();
const service = new PrometheusMetricsService(mock(), mock(), eventBus);
const service = new PrometheusMetricsService(mock(), eventBus);
await service.configureMetrics(mock<express.Application>());
await service.init(mock<express.Application>());
expect(eventBus.on).toHaveBeenCalledWith(
'metrics.messageEventBus.Event',
expect.any(Function),
);
expect(eventBus.on).toHaveBeenCalledWith('metrics.eventBus.event', expect.any(Function));
});
});
});

View file

@ -5,79 +5,116 @@ import promBundle from 'express-prom-bundle';
import promClient, { type Counter } from 'prom-client';
import semverParse from 'semver/functions/parse';
import { Service } from 'typedi';
import EventEmitter from 'events';
import { CacheService } from '@/services/cache/cache.service';
import { type EventMessageTypes } from '@/eventbus';
import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
import { Logger } from '@/Logger';
import { EventMessageTypeNames } from 'n8n-workflow';
import type { EventMessageTypes } from '@/eventbus';
type MetricCategory = 'default' | 'api' | 'cache' | 'logs';
@Service()
export class PrometheusMetricsService extends EventEmitter {
export class PrometheusMetricsService {
constructor(
private readonly logger: Logger,
private readonly cacheService: CacheService,
private readonly eventBus: MessageEventBus,
) {
super();
}
) {}
counters: Record<string, Counter<string> | null> = {};
private readonly counters: { [key: string]: Counter<string> | null } = {};
async configureMetrics(app: express.Application) {
private readonly prefix = config.getEnv('endpoints.metrics.prefix');
private readonly includes = {
metrics: {
default: config.getEnv('endpoints.metrics.includeDefaultMetrics'),
api: config.getEnv('endpoints.metrics.includeApiEndpoints'),
cache: config.getEnv('endpoints.metrics.includeCacheMetrics'),
logs: config.getEnv('endpoints.metrics.includeMessageEventBusMetrics'),
},
labels: {
credentialsType: config.getEnv('endpoints.metrics.includeCredentialTypeLabel'),
nodeType: config.getEnv('endpoints.metrics.includeNodeTypeLabel'),
workflowId: config.getEnv('endpoints.metrics.includeWorkflowIdLabel'),
apiPath: config.getEnv('endpoints.metrics.includeApiPathLabel'),
apiMethod: config.getEnv('endpoints.metrics.includeApiMethodLabel'),
apiStatusCode: config.getEnv('endpoints.metrics.includeApiStatusCodeLabel'),
},
};
async init(app: express.Application) {
promClient.register.clear(); // clear all metrics in case we call this a second time
this.setupDefaultMetrics();
this.setupN8nVersionMetric();
this.setupCacheMetrics();
this.setupMessageEventBusMetrics();
this.setupApiMetrics(app);
this.initDefaultMetrics();
this.initN8nVersionMetric();
this.initCacheMetrics();
this.initEventBusMetrics();
this.initApiMetrics(app);
this.mountMetricsEndpoint(app);
}
private setupN8nVersionMetric() {
const n8nVersion = semverParse(N8N_VERSION || '0.0.0');
enableMetric(metric: MetricCategory) {
this.includes.metrics[metric] = true;
}
disableMetric(metric: MetricCategory) {
this.includes.metrics[metric] = false;
}
disableAllMetrics() {
for (const metric of Object.keys(this.includes.metrics) as MetricCategory[]) {
this.includes.metrics[metric] = false;
}
}
/**
* Set up metric for n8n version: `n8n_version_info`
*/
private initN8nVersionMetric() {
const n8nVersion = semverParse(N8N_VERSION ?? '0.0.0');
if (!n8nVersion) return;
if (n8nVersion) {
const versionGauge = new promClient.Gauge({
name: config.getEnv('endpoints.metrics.prefix') + 'version_info',
name: this.prefix + 'version_info',
help: 'n8n version info.',
labelNames: ['version', 'major', 'minor', 'patch'],
});
versionGauge.set(
{
version: 'v' + n8nVersion.version,
major: n8nVersion.major,
minor: n8nVersion.minor,
patch: n8nVersion.patch,
},
1,
);
}
const { version, major, minor, patch } = n8nVersion;
versionGauge.set({ version: 'v' + version, major, minor, patch }, 1);
}
private setupDefaultMetrics() {
if (config.getEnv('endpoints.metrics.includeDefaultMetrics')) {
/**
* Set up default metrics collection with `prom-client`, e.g.
* `process_cpu_seconds_total`, `process_resident_memory_bytes`, etc.
*/
private initDefaultMetrics() {
if (!this.includes.metrics.default) return;
promClient.collectDefaultMetrics();
}
}
private setupApiMetrics(app: express.Application) {
if (config.getEnv('endpoints.metrics.includeApiEndpoints')) {
/**
* Set up metrics for API endpoints with `express-prom-bundle`
*/
private initApiMetrics(app: express.Application) {
if (!this.includes.metrics.api) return;
const metricsMiddleware = promBundle({
autoregister: false,
includeUp: false,
includePath: config.getEnv('endpoints.metrics.includeApiPathLabel'),
includeMethod: config.getEnv('endpoints.metrics.includeApiMethodLabel'),
includeStatusCode: config.getEnv('endpoints.metrics.includeApiStatusCodeLabel'),
includePath: this.includes.labels.apiPath,
includeMethod: this.includes.labels.apiMethod,
includeStatusCode: this.includes.labels.apiStatusCode,
});
app.use(['/rest/', '/webhook/', 'webhook-test/', '/api/'], metricsMiddleware);
}
app.use(
['/rest/', '/webhook/', '/webhook-waiting/', '/form-waiting/', '/webhook-test/', '/api/'],
metricsMiddleware,
);
}
mountMetricsEndpoint(app: express.Application) {
private mountMetricsEndpoint(app: express.Application) {
app.get('/metrics', async (_req: express.Request, res: express.Response) => {
const metrics = await promClient.register.metrics();
res.setHeader('Content-Type', promClient.register.contentType);
@ -85,116 +122,100 @@ export class PrometheusMetricsService extends EventEmitter {
});
}
private setupCacheMetrics() {
if (!config.getEnv('endpoints.metrics.includeCacheMetrics')) {
return;
}
this.counters.cacheHitsTotal = new promClient.Counter({
name: config.getEnv('endpoints.metrics.prefix') + 'cache_hits_total',
help: 'Total number of cache hits.',
/**
* Set up cache metrics: `n8n_cache_hits_total`, `n8n_cache_misses_total`, and
* `n8n_cache_updates_total`
*/
private initCacheMetrics() {
if (!this.includes.metrics.cache) return;
const [hitsConfig, missesConfig, updatesConfig] = ['hits', 'misses', 'updates'].map((kind) => ({
name: this.prefix + 'cache_' + kind + '_total',
help: `Total number of cache ${kind}.`,
labelNames: ['cache'],
});
}));
this.counters.cacheHitsTotal = new promClient.Counter(hitsConfig);
this.counters.cacheHitsTotal.inc(0);
this.cacheService.on('metrics.cache.hit', (amount: number = 1) => {
this.counters.cacheHitsTotal?.inc(amount);
});
this.cacheService.on('metrics.cache.hit', () => this.counters.cacheHitsTotal?.inc(1));
this.counters.cacheMissesTotal = new promClient.Counter({
name: config.getEnv('endpoints.metrics.prefix') + 'cache_misses_total',
help: 'Total number of cache misses.',
labelNames: ['cache'],
});
this.counters.cacheMissesTotal = new promClient.Counter(missesConfig);
this.counters.cacheMissesTotal.inc(0);
this.cacheService.on('metrics.cache.miss', (amount: number = 1) => {
this.counters.cacheMissesTotal?.inc(amount);
});
this.cacheService.on('metrics.cache.miss', () => this.counters.cacheMissesTotal?.inc(1));
this.counters.cacheUpdatesTotal = new promClient.Counter({
name: config.getEnv('endpoints.metrics.prefix') + 'cache_updates_total',
help: 'Total number of cache updates.',
labelNames: ['cache'],
});
this.counters.cacheUpdatesTotal = new promClient.Counter(updatesConfig);
this.counters.cacheUpdatesTotal.inc(0);
this.cacheService.on('metrics.cache.update', (amount: number = 1) => {
this.counters.cacheUpdatesTotal?.inc(amount);
});
this.cacheService.on('metrics.cache.update', () => this.counters.cacheUpdatesTotal?.inc(1));
}
private getCounterForEvent(event: EventMessageTypes): Counter<string> | null {
if (!promClient) return null;
if (!this.counters[event.eventName]) {
const prefix = config.getEnv('endpoints.metrics.prefix');
const metricName =
prefix + event.eventName.replace('n8n.', '').replace(/\./g, '_') + '_total';
private toCounter(event: EventMessageTypes) {
const { eventName } = event;
if (!this.counters[eventName]) {
const metricName = this.prefix + eventName.replace('n8n.', '').replace(/\./g, '_') + '_total';
if (!promClient.validateMetricName(metricName)) {
this.logger.debug(`Invalid metric name: ${metricName}. Ignoring it!`);
this.counters[event.eventName] = null;
this.counters[eventName] = null;
return null;
}
const labels = this.toLabels(event);
const counter = new promClient.Counter({
name: metricName,
help: `Total number of ${event.eventName} events.`,
labelNames: Object.keys(this.getLabelsForEvent(event)),
help: `Total number of ${eventName} events.`,
labelNames: Object.keys(labels),
});
counter.inc(0);
this.counters[event.eventName] = counter;
counter.labels(labels).inc(0);
this.counters[eventName] = counter;
}
return this.counters[event.eventName];
return this.counters[eventName];
}
private setupMessageEventBusMetrics() {
if (!config.getEnv('endpoints.metrics.includeMessageEventBusMetrics')) {
return;
}
this.eventBus.on('metrics.messageEventBus.Event', (event: EventMessageTypes) => {
const counter = this.getCounterForEvent(event);
private initEventBusMetrics() {
if (!this.includes.metrics.logs) return;
this.eventBus.on('metrics.eventBus.event', (event: EventMessageTypes) => {
const counter = this.toCounter(event);
if (!counter) return;
counter.inc(1);
});
}
getLabelsForEvent(event: EventMessageTypes): Record<string, string> {
switch (event.__type) {
private toLabels(event: EventMessageTypes): Record<string, string> {
const { __type, eventName, payload } = event;
switch (__type) {
case EventMessageTypeNames.audit:
if (event.eventName.startsWith('n8n.audit.user.credentials')) {
return config.getEnv('endpoints.metrics.includeCredentialTypeLabel')
? {
credential_type: this.getLabelValueForCredential(
event.payload.credentialType ?? 'unknown',
),
}
if (eventName.startsWith('n8n.audit.user.credentials')) {
return this.includes.labels.credentialsType
? { credential_type: (event.payload.credentialType ?? 'unknown').replace(/\./g, '_') }
: {};
}
if (event.eventName.startsWith('n8n.audit.workflow')) {
return config.getEnv('endpoints.metrics.includeWorkflowIdLabel')
? { workflow_id: event.payload.workflowId?.toString() ?? 'unknown' }
if (eventName.startsWith('n8n.audit.workflow')) {
return this.includes.labels.workflowId
? { workflow_id: payload.workflowId ?? 'unknown' }
: {};
}
break;
case EventMessageTypeNames.node:
return config.getEnv('endpoints.metrics.includeNodeTypeLabel')
? { node_type: this.getLabelValueForNode(event.payload.nodeType ?? 'unknown') }
return this.includes.labels.nodeType
? {
node_type: (payload.nodeType ?? 'unknown')
.replace('n8n-nodes-', '')
.replace(/\./g, '_'),
}
: {};
case EventMessageTypeNames.workflow:
return config.getEnv('endpoints.metrics.includeWorkflowIdLabel')
? { workflow_id: event.payload.workflowId?.toString() ?? 'unknown' }
return this.includes.labels.workflowId
? { workflow_id: payload.workflowId ?? 'unknown' }
: {};
}
return {};
}
getLabelValueForNode(nodeType: string) {
return nodeType.replace('n8n-nodes-', '').replace(/\./g, '_');
}
getLabelValueForCredential(credentialType: string) {
return credentialType.replace(/\./g, '_');
}
}

View file

@ -1,83 +0,0 @@
import { Container } from 'typedi';
import { parse as semverParse } from 'semver';
import request from 'supertest';
import config from '@/config';
import { N8N_VERSION } from '@/constants';
import { PrometheusMetricsService } from '@/metrics/prometheus-metrics.service';
import { ExecutionRecoveryService } from '@/executions/execution-recovery.service';
import { setupTestServer } from './shared/utils';
import { mockInstance } from '../shared/mocking';
mockInstance(ExecutionRecoveryService);
jest.unmock('@/eventbus/MessageEventBus/MessageEventBus');
config.set('endpoints.metrics.enable', true);
config.set('endpoints.metrics.includeDefaultMetrics', false);
config.set('endpoints.metrics.prefix', 'n8n_test_');
const testServer = setupTestServer({ endpointGroups: ['metrics'] });
let testAgent = request.agent(testServer.app);
async function getMetricsResponseAsLines() {
const response = await testAgent.get('/metrics');
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
const lines = response.text.trim().split('\n');
return lines;
}
describe('Metrics', () => {
it('should return n8n version', async () => {
const n8nVersion = semverParse(N8N_VERSION || '0.0.0');
expect(n8nVersion).toBeTruthy();
const lines = await getMetricsResponseAsLines();
expect(lines).toContain(
`n8n_test_version_info{version="v${n8nVersion!.version}",major="${
n8nVersion!.major
}",minor="${n8nVersion!.minor}",patch="${n8nVersion!.patch}"} 1`,
);
});
it('should return cache metrics when enabled', async () => {
config.set('endpoints.metrics.includeCacheMetrics', true);
await Container.get(PrometheusMetricsService).configureMetrics(testServer.app);
const lines = await getMetricsResponseAsLines();
expect(lines).toContain('n8n_test_cache_hits_total 0');
expect(lines).toContain('n8n_test_cache_misses_total 0');
expect(lines).toContain('n8n_test_cache_updates_total 0');
});
// TODO: Commented out due to flakiness in CI
// it('should return event metrics when enabled', async () => {
// config.set('endpoints.metrics.includeMessageEventBusMetrics', true);
// await Container.get(MetricsService).configureMetrics(testServer.app);
// await eventBus.initialize();
// await eventBus.send(
// new EventMessageGeneric({
// eventName: 'n8n.destination.test',
// }),
// );
// const lines = await getMetricsResponseAsLines();
// expect(lines).toContain('n8n_test_destination_test_total 1');
// await eventBus.close();
// jest.mock('@/eventbus/MessageEventBus/MessageEventBus');
// });
it('should return default metrics', async () => {
config.set('endpoints.metrics.includeDefaultMetrics', true);
await Container.get(PrometheusMetricsService).configureMetrics(testServer.app);
const lines = await getMetricsResponseAsLines();
expect(lines).toContain('nodejs_heap_space_size_total_bytes{space="read_only"} 0');
config.set('endpoints.metrics.includeDefaultMetrics', false);
});
it('should not return default metrics only when disabled', async () => {
config.set('endpoints.metrics.includeDefaultMetrics', false);
await Container.get(PrometheusMetricsService).configureMetrics(testServer.app);
const lines = await getMetricsResponseAsLines();
expect(lines).not.toContain('nodejs_heap_space_size_total_bytes{space="read_only"} 0');
config.set('endpoints.metrics.includeDefaultMetrics', true);
});
});

View file

@ -0,0 +1,149 @@
import { Container } from 'typedi';
import { parse as semverParse } from 'semver';
import request, { type Response } from 'supertest';
import config from '@/config';
import { N8N_VERSION } from '@/constants';
import { PrometheusMetricsService } from '@/metrics/prometheus-metrics.service';
import { setupTestServer } from './shared/utils';
jest.unmock('@/eventbus/MessageEventBus/MessageEventBus');
const toLines = (response: Response) => response.text.trim().split('\n');
config.set('endpoints.metrics.enable', true);
config.set('endpoints.metrics.prefix', 'n8n_test_');
const server = setupTestServer({ endpointGroups: ['metrics'] });
const agent = request.agent(server.app);
let prometheusService: PrometheusMetricsService;
describe('Metrics', () => {
beforeAll(() => {
prometheusService = Container.get(PrometheusMetricsService);
});
beforeEach(() => {
prometheusService.disableAllMetrics();
});
it('should return n8n version', async () => {
/**
* Arrange
*/
await prometheusService.init(server.app);
/**
* Act
*/
const response = await agent.get('/metrics');
/**
* Assert
*/
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
const n8nVersion = semverParse(N8N_VERSION);
if (!n8nVersion) fail('Failed to parse n8n version');
const { version, major, minor, patch } = n8nVersion;
expect(toLines(response)).toContain(
`n8n_test_version_info{version="v${version}",major="${major}",minor="${minor}",patch="${patch}"} 1`,
);
});
it('should return default metrics if enabled', async () => {
/**
* Arrange
*/
prometheusService.enableMetric('default');
await prometheusService.init(server.app);
/**
* Act
*/
const response = await agent.get('/metrics');
/**
* Assert
*/
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
expect(toLines(response)).toContain('nodejs_heap_space_size_total_bytes{space="read_only"} 0');
});
it('should not return default metrics if disabled', async () => {
/**
* Arrange
*/
prometheusService.disableMetric('default');
await prometheusService.init(server.app);
/**
* Act
*/
const response = await agent.get('/metrics');
/**
* Assert
*/
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
expect(toLines(response)).not.toContain(
'nodejs_heap_space_size_total_bytes{space="read_only"} 0',
);
});
it('should return cache metrics if enabled', async () => {
/**
* Arrange
*/
prometheusService.enableMetric('cache');
await prometheusService.init(server.app);
/**
* Act
*/
const response = await agent.get('/metrics');
/**
* Assert
*/
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
const lines = toLines(response);
expect(lines).toContain('n8n_test_cache_hits_total 0');
expect(lines).toContain('n8n_test_cache_misses_total 0');
expect(lines).toContain('n8n_test_cache_updates_total 0');
});
it('should not return cache metrics if disabled', async () => {
/**
* Arrange
*/
await prometheusService.init(server.app);
/**
* Act
*/
const response = await agent.get('/metrics');
/**
* Assert
*/
expect(response.status).toEqual(200);
expect(response.type).toEqual('text/plain');
const lines = toLines(response);
expect(lines).not.toContain('n8n_test_cache_hits_total 0');
expect(lines).not.toContain('n8n_test_cache_misses_total 0');
expect(lines).not.toContain('n8n_test_cache_updates_total 0');
});
});

View file

@ -148,7 +148,7 @@ export const setupTestServer = ({
const { PrometheusMetricsService } = await import(
'@/metrics/prometheus-metrics.service'
);
await Container.get(PrometheusMetricsService).configureMetrics(app);
await Container.get(PrometheusMetricsService).init(app);
break;
case 'eventBus':