From 643d28cebc520fffd21ae54fa15f162e1133221d Mon Sep 17 00:00:00 2001 From: Thomas Spalinger Date: Sat, 24 Aug 2024 18:02:57 +0200 Subject: [PATCH] Extend Prometheus Labels to include tags (requires restart for NEW labels on the monitor to be visible) (#4704) Co-authored-by: Frank Elsinga --- server/model/monitor.js | 4 +- server/prometheus.js | 144 ++++++++++++++++++++++++++++++---------- 2 files changed, 110 insertions(+), 38 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index a5c290c3b..b929e705c 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -336,7 +336,7 @@ class Monitor extends BeanModel { let previousBeat = null; let retries = 0; - this.prometheus = new Prometheus(this); + this.prometheus = await Prometheus.createAndInitMetrics(this); const beat = async () => { @@ -980,7 +980,7 @@ class Monitor extends BeanModel { await R.store(bean); log.debug("monitor", `[${this.name}] prometheus.update`); - this.prometheus?.update(bean, tlsInfo); + await this.prometheus?.update(bean, tlsInfo); previousBeat = bean; diff --git a/server/prometheus.js b/server/prometheus.js index f26125d2c..05a028397 100644 --- a/server/prometheus.js +++ b/server/prometheus.js @@ -1,3 +1,4 @@ +const { R } = require("redbean-node"); const PrometheusClient = require("prom-client"); const { log } = require("../src/util"); @@ -9,36 +10,102 @@ const commonLabels = [ "monitor_port", ]; -const monitorCertDaysRemaining = new PrometheusClient.Gauge({ - name: "monitor_cert_days_remaining", - help: "The number of days remaining until the certificate expires", - labelNames: commonLabels -}); - -const monitorCertIsValid = new PrometheusClient.Gauge({ - name: "monitor_cert_is_valid", - help: "Is the certificate still valid? (1 = Yes, 0= No)", - labelNames: commonLabels -}); -const monitorResponseTime = new PrometheusClient.Gauge({ - name: "monitor_response_time", - help: "Monitor Response Time (ms)", - labelNames: commonLabels -}); - -const monitorStatus = new PrometheusClient.Gauge({ - name: "monitor_status", - help: "Monitor Status (1 = UP, 0= DOWN, 2= PENDING, 3= MAINTENANCE)", - labelNames: commonLabels -}); - class Prometheus { - monitorLabelValues = {}; /** - * @param {object} monitor Monitor object to monitor + * Metric: monitor_cert_days_remaining + * @type {PrometheusClient.Gauge | null} */ - constructor(monitor) { + static monitorCertDaysRemaining = null; + + /** + * Metric: monitor_cert_is_valid + * @type {PrometheusClient.Gauge | null} + */ + static monitorCertIsValid = null; + + /** + * Metric: monitor_response_time + * @type {PrometheusClient.Gauge | null} + */ + static monitorResponseTime = null; + + /** + * Metric: monitor_status + * @type {PrometheusClient.Gauge | null} + */ + static monitorStatus = null; + + /** + * All registered metric labels. + * @type {string[] | null} + */ + static monitorLabelNames = null; + + /** + * Monitor labels/values combination. + * @type {{}} + */ + monitorLabelValues; + + /** + * Initialize metrics and get all label names the first time called. + * @returns {void} + */ + static async initMetrics() { + if (!this.monitorLabelNames) { + let labelNames = await R.getCol("SELECT name FROM tag"); + this.monitorLabelNames = [ ...commonLabels, ...labelNames ]; + } + if (!this.monitorCertDaysRemaining) { + this.monitorCertDaysRemaining = new PrometheusClient.Gauge({ + name: "monitor_cert_days_remaining", + help: "The number of days remaining until the certificate expires", + labelNames: this.monitorLabelNames + }); + } + if (!this.monitorCertIsValid) { + this.monitorCertIsValid = new PrometheusClient.Gauge({ + name: "monitor_cert_is_valid", + help: "Is the certificate still valid? (1 = Yes, 0 = No)", + labelNames: this.monitorLabelNames + }); + } + if (!this.monitorResponseTime) { + this.monitorResponseTime = new PrometheusClient.Gauge({ + name: "monitor_response_time", + help: "Monitor Response Time (ms)", + labelNames: this.monitorLabelNames + }); + } + if (!this.monitorStatus) { + this.monitorStatus = new PrometheusClient.Gauge({ + name: "monitor_status", + help: "Monitor Status (1 = UP, 0 = DOWN, 2 = PENDING, 3 = MAINTENANCE)", + labelNames: this.monitorLabelNames + }); + } + } + + /** + * Wrapper to create a `Prometheus` instance and ensure metrics are initialized. + * @param {Monitor} monitor Monitor object to monitor + * @returns {Promise} `Prometheus` instance + */ + static async createAndInitMetrics(monitor) { + await Prometheus.initMetrics(); + let tags = await monitor.getTags(); + return new Prometheus(monitor, tags); + } + + /** + * Creates a prometheus metric instance. + * + * Note: Make sure to call `Prometheus.initMetrics()` once prior creating Prometheus instances. + * @param {Monitor} monitor Monitor object to monitor + * @param {Promise[]>} tags Tags of the monitor + */ + constructor(monitor, tags) { this.monitorLabelValues = { monitor_name: monitor.name, monitor_type: monitor.type, @@ -46,6 +113,12 @@ class Prometheus { monitor_hostname: monitor.hostname, monitor_port: monitor.port }; + Object.values(tags) + // only label names that were known at first metric creation. + .filter(tag => Prometheus.monitorLabelNames.includes(tag.name)) + .forEach(tag => { + this.monitorLabelValues[tag.name] = tag.value; + }); } /** @@ -55,7 +128,6 @@ class Prometheus { * @returns {void} */ update(heartbeat, tlsInfo) { - if (typeof tlsInfo !== "undefined") { try { let isValid; @@ -64,7 +136,7 @@ class Prometheus { } else { isValid = 0; } - monitorCertIsValid.set(this.monitorLabelValues, isValid); + Prometheus.monitorCertIsValid.set(this.monitorLabelValues, isValid); } catch (e) { log.error("prometheus", "Caught error"); log.error("prometheus", e); @@ -72,7 +144,7 @@ class Prometheus { try { if (tlsInfo.certInfo != null) { - monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining); + Prometheus.monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining); } } catch (e) { log.error("prometheus", "Caught error"); @@ -82,7 +154,7 @@ class Prometheus { if (heartbeat) { try { - monitorStatus.set(this.monitorLabelValues, heartbeat.status); + Prometheus.monitorStatus.set(this.monitorLabelValues, heartbeat.status); } catch (e) { log.error("prometheus", "Caught error"); log.error("prometheus", e); @@ -90,10 +162,10 @@ class Prometheus { try { if (typeof heartbeat.ping === "number") { - monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping); + Prometheus.monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping); } else { // Is it good? - monitorResponseTime.set(this.monitorLabelValues, -1); + Prometheus.monitorResponseTime.set(this.monitorLabelValues, -1); } } catch (e) { log.error("prometheus", "Caught error"); @@ -108,10 +180,10 @@ class Prometheus { */ remove() { try { - monitorCertDaysRemaining.remove(this.monitorLabelValues); - monitorCertIsValid.remove(this.monitorLabelValues); - monitorResponseTime.remove(this.monitorLabelValues); - monitorStatus.remove(this.monitorLabelValues); + Prometheus.monitorCertDaysRemaining?.remove(this.monitorLabelValues); + Prometheus.monitorCertIsValid?.remove(this.monitorLabelValues); + Prometheus.monitorResponseTime?.remove(this.monitorLabelValues); + Prometheus.monitorStatus?.remove(this.monitorLabelValues); } catch (e) { console.error(e); }