From d3f60cc5ff1a635537dcf0560f168d54a00c59e8 Mon Sep 17 00:00:00 2001 From: Gleb Tcivie Date: Fri, 26 Jul 2024 12:02:17 +0300 Subject: [PATCH 1/5] Added support for filtering specific types of messages from being reported --- .env | 6 ++++-- exporter/processor_base.py | 3 --- exporter/processors.py | 6 +++++- main.py | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 440e3d7..91ef300 100644 --- a/.env +++ b/.env @@ -29,9 +29,11 @@ MQTT_CALLBACK_API_VERSION=VERSION2 MESH_HIDE_SOURCE_DATA=false ## Hide destination data in the exporter (default: false) MESH_HIDE_DESTINATION_DATA=false -## Filtered ports in the exporter (default: 1, can be a comma-separated list of ports) -FILTERED_PORTS=0 ## Hide message content in the TEXT_MESSAGE_APP packets (default: true) (Currently we only log message length, if we hide then all messages would have the same length) HIDE_MESSAGE=false ## MQTT server Key for decoding MQTT_SERVER_KEY=1PG7OiApB1nwvP+rz05pAQ== + +# Message types to filter (default: none) (comma separated) (eg. TEXT_MESSAGE_APP,POSITION_APP) +# Full list can be found here: https://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum +EXPORTER_MESSAGE_TYPES_TO_FILTER=TEXT_MESSAGE_APP,POSITION_APP diff --git a/exporter/processor_base.py b/exporter/processor_base.py index 8d9939f..b2460e1 100644 --- a/exporter/processor_base.py +++ b/exporter/processor_base.py @@ -165,9 +165,6 @@ class MessageProcessor: short_name='Hidden', long_name='Hidden') - if port_num in map(int, os.getenv('FILTERED_PORTS', '1').split(',')): # Filter out ports - return None # Ignore this packet - self.process_simple_packet_details(destination_client_details, mesh_packet, port_num, source_client_details) processor = ProcessorRegistry.get_processor(port_num)(self.registry, self.db_pool) diff --git a/exporter/processors.py b/exporter/processors.py index 652c377..9d0339f 100644 --- a/exporter/processors.py +++ b/exporter/processors.py @@ -54,6 +54,11 @@ class ProcessorRegistry: @classmethod def register_processor(cls, port_num): def inner_wrapper(wrapped_class): + if PortNum.DESCRIPTOR.values_by_number[port_num].name in os.getenv('EXPORTER_MESSAGE_TYPES_TO_FILTER', + '').split(','): + logger.info(f"Processor for port_num {port_num} is filtered out") + return wrapped_class + cls._registry[port_num] = wrapped_class return wrapped_class @@ -71,7 +76,6 @@ class ProcessorRegistry: @ProcessorRegistry.register_processor(PortNum.UNKNOWN_APP) class UnknownAppProcessor(Processor): def process(self, payload: bytes, client_details: ClientDetails): - logger.debug("Received UNKNOWN_APP packet") return None diff --git a/main.py b/main.py index 1491c53..8574514 100644 --- a/main.py +++ b/main.py @@ -17,8 +17,6 @@ except ImportError: from prometheus_client import CollectorRegistry, start_http_server from psycopg_pool import ConnectionPool -from exporter.processor_base import MessageProcessor - connection_pool = None @@ -88,6 +86,9 @@ def handle_message(client, userdata, message): if __name__ == "__main__": load_dotenv() + # We have to load_dotenv before we can import MessageProcessor to allow filtering of message types + from exporter.processor_base import MessageProcessor + # Setup a connection pool connection_pool = ConnectionPool( os.getenv('DATABASE_URL'), From ea3f00b46690b7577624a5c28e2ce59a2d9ea5d9 Mon Sep 17 00:00:00 2001 From: Gleb Tcivie Date: Fri, 26 Jul 2024 13:00:59 +0300 Subject: [PATCH 2/5] Added more "Static" data like geolocation to the PostgressDB and removed it from prometheus to reduce the load. + Added support for lookup of Country + City + State for nodes per geolocation. --- docker-compose.yml | 2 +- docker/postgres/init.sql | 36 ++++++++++++++----------- exporter/db_handler.py | 17 ++++++++++++ exporter/processor_base.py | 4 +-- exporter/processors.py | 43 +++++++++++------------------- exporter/registry.py | 54 ++++++++++++++++++-------------------- main.py | 2 +- 7 files changed, 83 insertions(+), 75 deletions(-) create mode 100644 exporter/db_handler.py diff --git a/docker-compose.yml b/docker-compose.yml index 7eef747..abec137 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: - mesh-bridge postgres: - image: postgres:13.3 + image: postgres:16.3 restart: unless-stopped networks: - mesh-bridge diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql index 7cd9c5a..2ae31b6 100644 --- a/docker/postgres/init.sql +++ b/docker/postgres/init.sql @@ -19,23 +19,26 @@ CREATE TRIGGER trigger_expire_old_messages FOR EACH ROW EXECUTE FUNCTION expire_old_messages(); -CREATE TABLE IF NOT EXISTS client_details +CREATE TABLE IF NOT EXISTS node_details ( node_id VARCHAR PRIMARY KEY, +-- Base Data short_name VARCHAR, long_name VARCHAR, hardware_model VARCHAR, role VARCHAR, - mqtt_status VARCHAR default 'none' -); - -CREATE TABLE IF NOT EXISTS node_graph -( - node_id VARCHAR PRIMARY KEY, - last_sent_by_node_id VARCHAR, - last_sent_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - broadcast_interval_secs INTEGER, - FOREIGN KEY (node_id) REFERENCES client_details (node_id) + mqtt_status VARCHAR default 'none', +-- Location Data + longitude FLOAT, + latitude FLOAT, + altitude FLOAT, + precision FLOAT, + country VARCHAR, + city VARCHAR, + state VARCHAR, +-- SQL Data + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ); CREATE TABLE IF NOT EXISTS node_neighbors @@ -44,12 +47,15 @@ CREATE TABLE IF NOT EXISTS node_neighbors node_id VARCHAR, neighbor_id VARCHAR, snr FLOAT, - FOREIGN KEY (node_id) REFERENCES client_details (node_id), - FOREIGN KEY (neighbor_id) REFERENCES client_details (node_id), + FOREIGN KEY (node_id) REFERENCES node_details (node_id), + FOREIGN KEY (neighbor_id) REFERENCES node_details (node_id), UNIQUE (node_id, neighbor_id) ); CREATE UNIQUE INDEX idx_unique_node_neighbor ON node_neighbors (node_id, neighbor_id); -ALTER TABLE client_details - ADD COLUMN IF NOT EXISTS created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP; +CREATE OR REPLACE TRIGGER client_details_updated_at + BEFORE UPDATE + ON node_details + FOR EACH ROW +EXECUTE PROCEDURE moddatetime(updated_at); diff --git a/exporter/db_handler.py b/exporter/db_handler.py new file mode 100644 index 0000000..33a93a1 --- /dev/null +++ b/exporter/db_handler.py @@ -0,0 +1,17 @@ +from psycopg_pool import ConnectionPool + + +class DBHandler: + def __init__(self, db_pool: ConnectionPool): + self.db_pool = db_pool + + def get_connection(self): + return self.db_pool.getconn() + + def release_connection(self, conn): + self.db_pool.putconn(conn) + + def execute_db_operation(self, operation): + with self.db_pool.connection() as conn: + with conn.cursor() as cur: + return operation(cur, conn) diff --git a/exporter/processor_base.py b/exporter/processor_base.py index b2460e1..b66ba4b 100644 --- a/exporter/processor_base.py +++ b/exporter/processor_base.py @@ -258,7 +258,7 @@ class MessageProcessor: # First, try to select the existing record cur.execute(""" SELECT node_id, short_name, long_name, hardware_model, role - FROM client_details + FROM node_details WHERE node_id = %s; """, (node_id_str,)) result = cur.fetchone() @@ -266,7 +266,7 @@ class MessageProcessor: if not result: # If the client is not found, insert a new record cur.execute(""" - INSERT INTO client_details (node_id, short_name, long_name, hardware_model, role) + INSERT INTO node_details (node_id, short_name, long_name, hardware_model, role) VALUES (%s, %s, %s, %s, %s) RETURNING node_id, short_name, long_name, hardware_model, role; """, (node_id_str, 'Unknown', 'Unknown', HardwareModel.UNSET, None)) diff --git a/exporter/processors.py b/exporter/processors.py index 9d0339f..eb9b269 100644 --- a/exporter/processors.py +++ b/exporter/processors.py @@ -5,6 +5,8 @@ from venv import logger import psycopg import unishox2 +from exporter.db_handler import DBHandler + try: from meshtastic.admin_pb2 import AdminMessage from meshtastic.mesh_pb2 import Position, User, HardwareModel, Routing, Waypoint, RouteDiscovery, NeighborInfo @@ -36,17 +38,12 @@ from exporter.registry import _Metrics class Processor(ABC): def __init__(self, registry: CollectorRegistry, db_pool: ConnectionPool): self.db_pool = db_pool - self.metrics = _Metrics(registry) + self.metrics = _Metrics(registry, DBHandler(db_pool)) @abstractmethod def process(self, payload: bytes, client_details: ClientDetails): pass - def execute_db_operation(self, operation): - with self.db_pool.connection() as conn: - with conn.cursor() as cur: - return operation(cur, conn) - class ProcessorRegistry: _registry = {} @@ -113,18 +110,10 @@ class PositionAppProcessor(Processor): except Exception as e: logger.error(f"Failed to parse POSITION_APP packet: {e}") return - self.metrics.device_latitude_gauge.labels( - **client_details.to_dict() - ).set(position.latitude_i) - self.metrics.device_longitude_gauge.labels( - **client_details.to_dict() - ).set(position.longitude_i) - self.metrics.device_altitude_gauge.labels( - **client_details.to_dict() - ).set(position.altitude) - self.metrics.device_position_precision_gauge.labels( - **client_details.to_dict() - ).set(position.precision_bits) + + self.metrics.update_metrics_position( + position.latitude_i, position.longitude_i, position.altitude, + position.precision_bits, client_details) pass @@ -143,7 +132,7 @@ class NodeInfoAppProcessor(Processor): # First, try to select the existing record cur.execute(""" SELECT short_name, long_name, hardware_model, role - FROM client_details + FROM node_details WHERE node_id = %s; """, (client_details.node_id,)) existing_record = cur.fetchone() @@ -167,7 +156,7 @@ class NodeInfoAppProcessor(Processor): if update_fields: update_query = f""" - UPDATE client_details + UPDATE node_details SET {", ".join(update_fields)} WHERE node_id = %s """ @@ -175,7 +164,7 @@ class NodeInfoAppProcessor(Processor): else: # If record doesn't exist, insert a new one cur.execute(""" - INSERT INTO client_details (node_id, short_name, long_name, hardware_model, role) + INSERT INTO node_details (node_id, short_name, long_name, hardware_model, role) VALUES (%s, %s, %s, %s, %s) """, (client_details.node_id, user.short_name, user.long_name, ClientDetails.get_hardware_model_name_from_code(user.hw_model), @@ -183,7 +172,7 @@ class NodeInfoAppProcessor(Processor): conn.commit() - self.execute_db_operation(db_operation) + self.metrics.db.execute_db_operation(db_operation) @ProcessorRegistry.register_processor(PortNum.ROUTING_APP) @@ -531,7 +520,7 @@ class NeighborInfoAppProcessor(Processor): """, (client_details.node_id, neighbor_info.last_sent_by_id, neighbor_info.node_broadcast_interval_secs)) conn.commit() - self.execute_db_operation(operation) + self.metrics.db.execute_db_operation(operation) def update_node_neighbors(self, neighbor_info: NeighborInfo, client_details: ClientDetails): def operation(cur, conn): @@ -554,18 +543,18 @@ class NeighborInfoAppProcessor(Processor): DO UPDATE SET snr = EXCLUDED.snr RETURNING node_id, neighbor_id ) - INSERT INTO client_details (node_id) + INSERT INTO node_details (node_id) SELECT node_id FROM upsert - WHERE NOT EXISTS (SELECT 1 FROM client_details WHERE node_id = upsert.node_id) + WHERE NOT EXISTS (SELECT 1 FROM node_details WHERE node_id = upsert.node_id) UNION SELECT neighbor_id FROM upsert - WHERE NOT EXISTS (SELECT 1 FROM client_details WHERE node_id = upsert.neighbor_id) + WHERE NOT EXISTS (SELECT 1 FROM node_details WHERE node_id = upsert.neighbor_id) ON CONFLICT (node_id) DO NOTHING; """, (str(client_details.node_id), str(neighbor.node_id), float(neighbor.snr))) conn.commit() - self.execute_db_operation(operation) + self.metrics.db.execute_db_operation(operation) @ProcessorRegistry.register_processor(PortNum.ATAK_PLUGIN) diff --git a/exporter/registry.py b/exporter/registry.py index bf43091..99c591b 100644 --- a/exporter/registry.py +++ b/exporter/registry.py @@ -1,5 +1,10 @@ +import geopy.point +from geopy.geocoders import Nominatim from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram +from exporter.client_details import ClientDetails +from exporter.db_handler import DBHandler + class _Metrics: _instance = None @@ -9,11 +14,13 @@ class _Metrics: cls._instance = super(_Metrics, cls).__new__(cls) return cls._instance - def __init__(self, registry: CollectorRegistry): + def __init__(self, registry: CollectorRegistry, db: DBHandler): if not hasattr(self, 'initialized'): # Ensuring __init__ runs only once self._registry = registry self._init_metrics() self.initialized = True # Attribute to indicate initialization + self.geolocator = Nominatim() + self.db = db @staticmethod def _get_common_labels(): @@ -27,7 +34,6 @@ class _Metrics: self._init_metrics_telemetry_environment() self._init_metrics_telemetry_air_quality() self._init_metrics_telemetry_power() - self._init_metrics_position() self._init_route_discovery_metrics() def _init_metrics_text_message(self): @@ -38,31 +44,23 @@ class _Metrics: registry=self._registry ) - def _init_metrics_position(self): - self.device_latitude_gauge = Gauge( - 'device_latitude', - 'Device latitude', - self._get_common_labels(), - registry=self._registry - ) - self.device_longitude_gauge = Gauge( - 'device_longitude', - 'Device longitude', - self._get_common_labels(), - registry=self._registry - ) - self.device_altitude_gauge = Gauge( - 'device_altitude', - 'Device altitude', - self._get_common_labels(), - registry=self._registry - ) - self.device_position_precision_gauge = Gauge( - 'device_position_precision', - 'Device position precision', - self._get_common_labels(), - registry=self._registry - ) + def update_metrics_position(self, latitude, longitude, altitude, precision, client_details: ClientDetails): + point = geopy.point.Point(latitude, longitude, altitude) + location = self.geolocator.reverse(point, language='en') + + country = location.raw.get('address', {}).get('country', 'Unknown') + city = location.raw.get('address', {}).get('city', 'Unknown') + state = location.raw.get('address', {}).get('state', 'Unknown') + + def db_operation(cur, conn): + cur.execute(""" + UPDATE node_details + SET latitude = %s, longitude = %s, altitude = %s, precision = %s, country = %s, city = %s, state = %s + WHERE node_id = %s + """, (latitude, longitude, altitude, precision, country, city, state, client_details.node_id)) + conn.commit() + + self.db.execute_db_operation(db_operation) def _init_metrics_telemetry_power(self): self.ch1_voltage_gauge = Gauge( @@ -336,5 +334,3 @@ class _Metrics: self._get_common_labels() + ['response_type'], registry=self._registry ) - - diff --git a/main.py b/main.py index 8574514..9770f80 100644 --- a/main.py +++ b/main.py @@ -36,7 +36,7 @@ def handle_connect(client, userdata, flags, reason_code, properties): def update_node_status(node_number, status): with connection_pool.connection() as conn: with conn.cursor() as cur: - cur.execute("INSERT INTO client_details (node_id, mqtt_status) VALUES (%s, %s)" + cur.execute("INSERT INTO node_details (node_id, mqtt_status) VALUES (%s, %s)" "ON CONFLICT(node_id)" "DO UPDATE SET mqtt_status = %s", (node_number, status, status)) conn.commit() From 821056664e571c707094ed8fd4f1e617ec064232 Mon Sep 17 00:00:00 2001 From: Gleb Tcivie Date: Fri, 26 Jul 2024 13:18:09 +0300 Subject: [PATCH 3/5] Removed metric on message size in char len and replaced with overall size of packet by common labels and portnum --- .env | 2 -- exporter/processor_base.py | 17 ++++++++++++++++- exporter/processors.py | 7 +------ exporter/registry.py | 11 +---------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.env b/.env index 91ef300..3f1ca5e 100644 --- a/.env +++ b/.env @@ -29,8 +29,6 @@ MQTT_CALLBACK_API_VERSION=VERSION2 MESH_HIDE_SOURCE_DATA=false ## Hide destination data in the exporter (default: false) MESH_HIDE_DESTINATION_DATA=false -## Hide message content in the TEXT_MESSAGE_APP packets (default: true) (Currently we only log message length, if we hide then all messages would have the same length) -HIDE_MESSAGE=false ## MQTT server Key for decoding MQTT_SERVER_KEY=1PG7OiApB1nwvP+rz05pAQ== diff --git a/exporter/processor_base.py b/exporter/processor_base.py index b66ba4b..80eaed8 100644 --- a/exporter/processor_base.py +++ b/exporter/processor_base.py @@ -1,5 +1,6 @@ import base64 import os +import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes @@ -20,6 +21,7 @@ from exporter.processors import ProcessorRegistry class MessageProcessor: def __init__(self, registry: CollectorRegistry, db_pool: ConnectionPool): + self.message_size_in_bytes = None self.rx_rssi_gauge = None self.channel_counter = None self.packet_id_counter = None @@ -44,6 +46,13 @@ class MessageProcessor: 'destination_role' ] + self.message_size_in_bytes = Histogram( + 'text_message_app_size_in_bytes', + 'Size of text messages processed by the app in Bytes', + common_labels + ['portnum'], + registry=self.registry + ) + self.source_message_type_counter = Counter( 'mesh_packet_source_types', 'Types of mesh packets processed by source', @@ -181,7 +190,8 @@ class MessageProcessor: return enum_value.name return 'UNKNOWN_PORT' - def process_simple_packet_details(self, destination_client_details, mesh_packet, port_num, source_client_details): + def process_simple_packet_details(self, destination_client_details, mesh_packet: MeshPacket, port_num, + source_client_details): common_labels = { 'source_id': source_client_details.node_id, 'source_short_name': source_client_details.short_name, @@ -195,6 +205,11 @@ class MessageProcessor: 'destination_role': destination_client_details.role, } + self.message_size_in_bytes.labels( + **common_labels, + portnum=self.get_port_name_from_portnum(port_num) + ).observe(sys.getsizeof(mesh_packet)) + self.source_message_type_counter.labels( **common_labels, portnum=self.get_port_name_from_portnum(port_num) diff --git a/exporter/processors.py b/exporter/processors.py index eb9b269..eacda36 100644 --- a/exporter/processors.py +++ b/exporter/processors.py @@ -80,12 +80,7 @@ class UnknownAppProcessor(Processor): class TextMessageAppProcessor(Processor): def process(self, payload: bytes, client_details: ClientDetails): logger.debug("Received TEXT_MESSAGE_APP packet") - message = payload.decode('utf-8') - if os.getenv('HIDE_MESSAGE', 'true') == 'true': - message = 'Hidden' - self.metrics.message_length_histogram.labels( - **client_details.to_dict() - ).observe(len(message)) + pass @ProcessorRegistry.register_processor(PortNum.REMOTE_HARDWARE_APP) diff --git a/exporter/registry.py b/exporter/registry.py index 99c591b..ec53d56 100644 --- a/exporter/registry.py +++ b/exporter/registry.py @@ -1,6 +1,6 @@ import geopy.point from geopy.geocoders import Nominatim -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram +from prometheus_client import CollectorRegistry, Counter, Gauge from exporter.client_details import ClientDetails from exporter.db_handler import DBHandler @@ -29,21 +29,12 @@ class _Metrics: ] def _init_metrics(self): - self._init_metrics_text_message() self._init_metrics_telemetry_device() self._init_metrics_telemetry_environment() self._init_metrics_telemetry_air_quality() self._init_metrics_telemetry_power() self._init_route_discovery_metrics() - def _init_metrics_text_message(self): - self.message_length_histogram = Histogram( - 'text_message_app_length', - 'Length of text messages processed by the app', - self._get_common_labels(), - registry=self._registry - ) - def update_metrics_position(self, latitude, longitude, altitude, precision, client_details: ClientDetails): point = geopy.point.Point(latitude, longitude, altitude) location = self.geolocator.reverse(point, language='en') From 3cfadccc277b2f72e891a233f64cab63492674cc Mon Sep 17 00:00:00 2001 From: Gleb Tcivie Date: Fri, 26 Jul 2024 13:18:09 +0300 Subject: [PATCH 4/5] Removed metric on message size in char len and replaced with overall size of packet by common labels and portnum --- .env | 4 +--- exporter/processor_base.py | 17 ++++++++++++++++- exporter/processors.py | 7 +------ exporter/registry.py | 11 +---------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.env b/.env index 91ef300..40d461b 100644 --- a/.env +++ b/.env @@ -29,11 +29,9 @@ MQTT_CALLBACK_API_VERSION=VERSION2 MESH_HIDE_SOURCE_DATA=false ## Hide destination data in the exporter (default: false) MESH_HIDE_DESTINATION_DATA=false -## Hide message content in the TEXT_MESSAGE_APP packets (default: true) (Currently we only log message length, if we hide then all messages would have the same length) -HIDE_MESSAGE=false ## MQTT server Key for decoding MQTT_SERVER_KEY=1PG7OiApB1nwvP+rz05pAQ== # Message types to filter (default: none) (comma separated) (eg. TEXT_MESSAGE_APP,POSITION_APP) # Full list can be found here: https://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum -EXPORTER_MESSAGE_TYPES_TO_FILTER=TEXT_MESSAGE_APP,POSITION_APP +EXPORTER_MESSAGE_TYPES_TO_FILTER=TEXT_MESSAGE_APP diff --git a/exporter/processor_base.py b/exporter/processor_base.py index b66ba4b..80eaed8 100644 --- a/exporter/processor_base.py +++ b/exporter/processor_base.py @@ -1,5 +1,6 @@ import base64 import os +import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes @@ -20,6 +21,7 @@ from exporter.processors import ProcessorRegistry class MessageProcessor: def __init__(self, registry: CollectorRegistry, db_pool: ConnectionPool): + self.message_size_in_bytes = None self.rx_rssi_gauge = None self.channel_counter = None self.packet_id_counter = None @@ -44,6 +46,13 @@ class MessageProcessor: 'destination_role' ] + self.message_size_in_bytes = Histogram( + 'text_message_app_size_in_bytes', + 'Size of text messages processed by the app in Bytes', + common_labels + ['portnum'], + registry=self.registry + ) + self.source_message_type_counter = Counter( 'mesh_packet_source_types', 'Types of mesh packets processed by source', @@ -181,7 +190,8 @@ class MessageProcessor: return enum_value.name return 'UNKNOWN_PORT' - def process_simple_packet_details(self, destination_client_details, mesh_packet, port_num, source_client_details): + def process_simple_packet_details(self, destination_client_details, mesh_packet: MeshPacket, port_num, + source_client_details): common_labels = { 'source_id': source_client_details.node_id, 'source_short_name': source_client_details.short_name, @@ -195,6 +205,11 @@ class MessageProcessor: 'destination_role': destination_client_details.role, } + self.message_size_in_bytes.labels( + **common_labels, + portnum=self.get_port_name_from_portnum(port_num) + ).observe(sys.getsizeof(mesh_packet)) + self.source_message_type_counter.labels( **common_labels, portnum=self.get_port_name_from_portnum(port_num) diff --git a/exporter/processors.py b/exporter/processors.py index eb9b269..eacda36 100644 --- a/exporter/processors.py +++ b/exporter/processors.py @@ -80,12 +80,7 @@ class UnknownAppProcessor(Processor): class TextMessageAppProcessor(Processor): def process(self, payload: bytes, client_details: ClientDetails): logger.debug("Received TEXT_MESSAGE_APP packet") - message = payload.decode('utf-8') - if os.getenv('HIDE_MESSAGE', 'true') == 'true': - message = 'Hidden' - self.metrics.message_length_histogram.labels( - **client_details.to_dict() - ).observe(len(message)) + pass @ProcessorRegistry.register_processor(PortNum.REMOTE_HARDWARE_APP) diff --git a/exporter/registry.py b/exporter/registry.py index 99c591b..ec53d56 100644 --- a/exporter/registry.py +++ b/exporter/registry.py @@ -1,6 +1,6 @@ import geopy.point from geopy.geocoders import Nominatim -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram +from prometheus_client import CollectorRegistry, Counter, Gauge from exporter.client_details import ClientDetails from exporter.db_handler import DBHandler @@ -29,21 +29,12 @@ class _Metrics: ] def _init_metrics(self): - self._init_metrics_text_message() self._init_metrics_telemetry_device() self._init_metrics_telemetry_environment() self._init_metrics_telemetry_air_quality() self._init_metrics_telemetry_power() self._init_route_discovery_metrics() - def _init_metrics_text_message(self): - self.message_length_histogram = Histogram( - 'text_message_app_length', - 'Length of text messages processed by the app', - self._get_common_labels(), - registry=self._registry - ) - def update_metrics_position(self, latitude, longitude, altitude, precision, client_details: ClientDetails): point = geopy.point.Point(latitude, longitude, altitude) location = self.geolocator.reverse(point, language='en') From 0b487336fbb7952800dcc565a20a37fc9cf0de16 Mon Sep 17 00:00:00 2001 From: Gleb Tcivie Date: Fri, 26 Jul 2024 20:04:22 +0300 Subject: [PATCH 5/5] Bug fixes: Changed Histograms to Gages and updated the dashboards --- .env | 2 +- .../dashboards/Main Dashboard.json | 890 +++++++++-------- .../dashboards/Node Dashboard.json | 927 ++++++++++-------- docker/postgres/init.sql | 13 +- exporter/processor_base.py | 23 +- exporter/processors.py | 20 +- exporter/registry.py | 38 +- requirements.txt | 3 +- 8 files changed, 1038 insertions(+), 878 deletions(-) diff --git a/.env b/.env index 40d461b..fce0291 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ MQTT_PORT=1883 MQTT_USERNAME=meshdev MQTT_PASSWORD=large4cats MQTT_KEEPALIVE=60 -MQTT_TOPIC='msh/israel/#' +MQTT_TOPIC='msh/#' MQTT_IS_TLS=false # MQTT protocol version (default: MQTTv5) the public MQTT server supports MQTTv311 diff --git a/docker/grafana/provisioning/dashboards/Main Dashboard.json b/docker/grafana/provisioning/dashboards/Main Dashboard.json index 1ffd1fe..e27982e 100644 --- a/docker/grafana/provisioning/dashboards/Main Dashboard.json +++ b/docker/grafana/provisioning/dashboards/Main Dashboard.json @@ -11,14 +11,40 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": true, + "tags": [ + "server" + ], + "type": "dashboard" + }, "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "light-yellow", + "name": "Annotations", + "target": { + "limit": 100, + "matchAny": true, + "tags": [ + "server", + "MQTT" + ], + "type": "tags" + } } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 11, + "id": 1, "links": [], "panels": [ { @@ -39,10 +65,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -83,7 +105,7 @@ }, "editorMode": "builder", "format": "table", - "rawSql": "SELECT COUNT(node_id) FROM client_details WHERE node_id <> '0' LIMIT 50 ", + "rawSql": "SELECT COUNT(node_id) FROM node_details WHERE node_id <> '0' LIMIT 50 ", "refId": "A", "sql": { "columns": [ @@ -129,12 +151,86 @@ }, "whereString": "node_id <> '0'" }, - "table": "client_details" + "table": "node_details" } ], "title": "Total nodes in mesh", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "description": "Nodes that sent any packet in the last 30 minutes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 4, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(count by(source_id) (count by(source_id) (delta(mesh_packet_ids_created{source_id=~\"$Nodes\"}[30m]))))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{source_long_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active nodes in the last 30 minutes", + "type": "stat" + }, { "datasource": { "type": "postgres", @@ -184,7 +280,7 @@ "gridPos": { "h": 4, "w": 5, - "x": 4, + "x": 9, "y": 0 }, "id": 17, @@ -214,7 +310,7 @@ }, "editorMode": "builder", "format": "table", - "rawSql": "SELECT COUNT(mqtt_status), mqtt_status FROM client_details WHERE (COALESCE(mqtt_status, '') <> '' AND mqtt_status NOT LIKE '%none%') GROUP BY mqtt_status LIMIT 50 ", + "rawSql": "SELECT COUNT(mqtt_status), mqtt_status FROM node_details WHERE (COALESCE(mqtt_status, '') <> '' AND mqtt_status NOT LIKE '%none%') GROUP BY mqtt_status LIMIT 50 ", "refId": "A", "sql": { "columns": [ @@ -292,7 +388,7 @@ }, "whereString": "(COALESCE(mqtt_status, '') <> '' AND mqtt_status NOT LIKE '%none%')" }, - "table": "client_details" + "table": "node_details" } ], "title": "MQTT Node status", @@ -319,14 +415,14 @@ } ] }, - "unit": "chars" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, - "x": 9, + "x": 14, "y": 0 }, "id": 6, @@ -356,109 +452,18 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(source_id) (mesh_packet_want_ack_total)", + "expr": "histogram_quantile(0.95, sum by(le) (rate(text_message_app_size_in_bytes_bucket{source_id=~\"$Nodes\"}[$__rate_interval])))", "fullMetaSearch": false, - "hide": true, + "hide": false, "includeNullMetadata": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(text_message_app_length_bucket{node_id=~\"$Nodes\"}[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{long_name}}", - "range": true, - "refId": "B", - "useBackend": false } ], - "title": "Typical Maximum Message Length", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Nodes that sent any packet in the last 30 minutes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 14, - "y": 0 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(count by(source_id) (count by(source_id) (delta(mesh_packet_ids_created{source_id=~\"$Nodes\"}[30m]))))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{source_long_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Active nodes in the last 30 minutes", + "title": "Typical Maximum Message Length in Bytes", "type": "stat" }, { @@ -551,15 +556,264 @@ }, { "datasource": { - "type": "postgres", + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "description": "Shows the max recorded chanel utilization and the average", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 11, + "axisSoftMin": 7, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 3600000, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 5 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max(telemetry_app_channel_utilization{node_id=~\"$Nodes\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Chanel utilization (Max)", + "range": true, + "refId": "Max Chanel Utilization recorded", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(telemetry_app_channel_utilization{node_id=~\"$Nodes\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Chanel utilization (Avg)", + "range": true, + "refId": "Average Chanel Utilization", + "useBackend": false + } + ], + "title": "Channel Utilization (ChUtil)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "description": "This panel shows the percentage of airtime used for transmissions in the last hour. Airtime in LoRa networks represents the duration a device occupies the radio frequency channel to send data. It's a critical metric for:\n\nNetwork capacity: Higher airtime usage indicates increased network load.\n\nRegulatory compliance: Many regions limit the total airtime per device.\n\nBattery life: More airtime generally means higher power consumption.\n\nThe data comes from Meshtastic packets, reflecting actual network usage. High percentages may suggest the need for optimization or capacity planning.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 6.5, + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 8 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 13, + "x": 11, + "y": 5 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "telemetry_app_air_util_tx{node_id=~\"$Nodes\"} > -100\nand\ndelta(telemetry_app_air_util_tx{node_id=~\"$Nodes\"}[1m]) != 0", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{long_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Duty Cycle (AirUtilTX)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", "uid": "PA942B37CCFAF5A81" }, "description": "Graph that is built from Neighbor Info reports and shows the signal strenth for each line", "gridPos": { - "h": 29, - "w": 6, + "h": 32, + "w": 11, "x": 0, - "y": 5 + "y": 14 }, "id": 20, "options": { @@ -580,7 +834,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n node_id AS \"id\", \n long_name AS \"title\", \n hardware_model AS \"detail__Hardware Detail\", \n role AS \"detail__Client Role\", \n mqtt_status AS \"detail__MQTT Status\", \n short_name AS \"subtitle\",\n CASE \n WHEN mqtt_status = 'online' THEN '#2ECC71' -- Green for online\n WHEN mqtt_status = 'offline' THEN '#E74C3C' -- Red for offline\n ELSE '#808080' -- Gray for none or any other status\n END AS \"color\"\nFROM \n client_details", + "rawSql": "SELECT DISTINCT\n cd.node_id AS \"id\",\n cd.long_name AS \"title\",\n cd.hardware_model AS \"detail__Hardware Detail\",\n cd.role AS \"detail__Client Role\",\n cd.mqtt_status AS \"detail__MQTT Status\",\n cd.short_name AS \"subtitle\",\n CASE\n WHEN cd.mqtt_status = 'online' THEN '#2ECC71' -- Green for online\n WHEN cd.mqtt_status = 'offline' THEN '#E74C3C' -- Red for offline\n ELSE '#808080' -- Gray for none or any other status\n END AS \"color\"\nFROM\n node_details cd\nLEFT JOIN (\n SELECT node_id FROM node_neighbors\n UNION\n SELECT neighbor_id FROM node_neighbors\n) nn ON cd.node_id = nn.node_id\nWHERE nn.node_id IS NOT NULL", "refId": "nodes", "sql": { "columns": [ @@ -655,7 +909,7 @@ ], "limit": 50 }, - "table": "client_details" + "table": "node_details" }, { "datasource": { @@ -726,117 +980,6 @@ "title": "Node Graph", "type": "nodeGraph" }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "This panel shows the percentage of airtime used for transmissions in the last hour. Airtime in LoRa networks represents the duration a device occupies the radio frequency channel to send data. It's a critical metric for:\n\nNetwork capacity: Higher airtime usage indicates increased network load.\n\nRegulatory compliance: Many regions limit the total airtime per device.\n\nBattery life: More airtime generally means higher power consumption.\n\nThe data comes from Meshtastic packets, reflecting actual network usage. High percentages may suggest the need for optimization or capacity planning.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 12 - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 18, - "x": 6, - "y": 5 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "telemetry_app_air_util_tx{node_id=~\"$Nodes\"} > -100\nand\ndelta(telemetry_app_air_util_tx{node_id=~\"$Nodes\"}[1m]) != 0", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{long_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Percent of airtime for transmission used within the last hour", - "type": "timeseries" - }, { "datasource": { "type": "datasource", @@ -865,13 +1008,14 @@ "gridPos": { "h": 10, "w": 5, - "x": 6, + "x": 11, "y": 14 }, "id": 5, "options": { "displayLabels": [ - "name" + "name", + "percent" ], "legend": { "displayMode": "list", @@ -923,7 +1067,10 @@ "color": { "mode": "thresholds" }, + "fieldMinMax": false, "mappings": [], + "max": 7, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -947,8 +1094,8 @@ }, "gridPos": { "h": 10, - "w": 13, - "x": 11, + "w": 8, + "x": 16, "y": 14 }, "id": 9, @@ -1064,7 +1211,7 @@ { "targetBlank": true, "title": "Go to node dashboard", - "url": "http://grafana.mesh-il.com:3000/d/edqo1uh0eglq8f/node-dashboard?orgId=1&var-nodeID=${__data.fields[0]}" + "url": "http://grafana.mesh-il.com:3000/d/edqo1uh0eglq8g/node-dashboard?orgId=1&var-nodeID=${__data.fields[0]}" } ] }, @@ -1076,13 +1223,37 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Record Created At" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsLocal" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client Role" + }, + "properties": [ + { + "id": "custom.width", + "value": 126 + } + ] } ] }, "gridPos": { - "h": 10, - "w": 18, - "x": 6, + "h": 22, + "w": 13, + "x": 11, "y": 24 }, "id": 14, @@ -1115,7 +1286,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT * FROM client_details WHERE node_id <> '0'", + "rawSql": "SELECT * FROM node_details WHERE node_id <> '0'", "refId": "A", "sql": { "columns": [ @@ -1164,7 +1335,7 @@ }, "whereString": "node_id <> '0'" }, - "table": "client_details" + "table": "node_details" } ], "title": "General Information", @@ -1185,6 +1356,7 @@ "includeByName": {}, "indexByName": {}, "renameByName": { + "created_at": "Record Created At", "hardware_model": "Hardware Model", "long_name": "Long name", "mqtt_status": "MQTT Status", @@ -1199,8 +1371,8 @@ }, { "datasource": { - "type": "datasource", - "uid": "-- Mixed --" + "type": "grafana-postgresql-datasource", + "uid": "PA942B37CCFAF5A81" }, "description": "Displays the nodes on the map", "fieldConfig": { @@ -1250,10 +1422,10 @@ "overrides": [] }, "gridPos": { - "h": 11, + "h": 25, "w": 24, "x": 0, - "y": 34 + "y": 46 }, "id": 8, "options": { @@ -1315,9 +1487,13 @@ } } }, + "filterData": { + "id": "byRefId", + "options": "Nodes" + }, "location": { - "latitude": "Value #Latitude", - "longitude": "Value #Longitude", + "latitude": "latitude_norm", + "longitude": "longitude_norm", "mode": "coords" }, "name": "Layer 1", @@ -1339,74 +1515,6 @@ }, "pluginVersion": "11.1.0", "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "device_longitude{node_id=~\"$Nodes\"} * 1e-7", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Longitude" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "device_latitude{node_id=~\"$Nodes\"} * 1e-7", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Latitude" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "device_position_precision{node_id=~\"$Nodes\"}", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Precision", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "device_altitude{node_id=~\"$Nodes\"}", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Elevation", - "useBackend": false - }, { "datasource": { "type": "postgres", @@ -1416,8 +1524,8 @@ "format": "table", "hide": false, "rawQuery": true, - "rawSql": "SELECT * FROM client_details", - "refId": "A", + "rawSql": "SELECT * FROM node_details WHERE longitude != 0 AND longitude IS NOT NULL", + "refId": "Nodes", "sql": { "columns": [ { @@ -1440,18 +1548,11 @@ ], "limit": 50 }, - "table": "client_details" + "table": "node_details" } ], "title": "Nodes map", "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "node_id", - "mode": "outer" - } - }, { "id": "organize", "options": { @@ -1500,122 +1601,47 @@ "short_name 1": "Short Name" } } + }, + { + "id": "calculateField", + "options": { + "alias": "longitude_norm", + "binary": { + "left": "longitude", + "operator": "*", + "right": "1e-7" + }, + "mode": "binary", + "reduce": { + "include": [ + "longitude", + "latitude" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "latitude_norm", + "binary": { + "left": "latitude", + "operator": "*", + "right": "1e-7" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } } ], "type": "geomap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Last reported battely value of the nodes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 20 - }, - { - "color": "green", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "telemetry_app_battery_level{node_id=~\"$Nodes\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{long_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Nodes Battery report", - "type": "timeseries" } ], - "refresh": "1m", + "refresh": "5m", "schemaVersion": 39, "tags": [], "templating": { @@ -1630,14 +1656,14 @@ "type": "postgres", "uid": "PA942B37CCFAF5A81" }, - "definition": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM client_details \nORDER BY long_name", + "definition": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM node_details \nORDER BY long_name", "hide": 0, "includeAll": true, "label": "Nodes", "multi": true, "name": "Nodes", "options": [], - "query": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM client_details \nORDER BY long_name", + "query": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM node_details \nORDER BY long_name", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1647,13 +1673,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Main Dashboard", "uid": "edqkge9mf7v28g", - "version": 2, + "version": 5, "weekStart": "sunday" } \ No newline at end of file diff --git a/docker/grafana/provisioning/dashboards/Node Dashboard.json b/docker/grafana/provisioning/dashboards/Node Dashboard.json index d4c99cd..5ca02d1 100644 --- a/docker/grafana/provisioning/dashboards/Node Dashboard.json +++ b/docker/grafana/provisioning/dashboards/Node Dashboard.json @@ -18,40 +18,36 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 10, + "id": 2, "links": [], "panels": [ { "datasource": { - "type": "postgres", - "uid": "PA942B37CCFAF5A81" + "type": "datasource", + "uid": "-- Mixed --" }, - "description": "This scoring system evaluates nodes in our mesh network based on two key factors:\n\n* Importance Score (0-100)\nMeasures how critical the node is for overall network connectivity.\nA higher score indicates that the node is on many shortest paths between other nodes in the network.\nNodes with high importance scores are crucial for maintaining efficient network communication and resilience.\n\n* Stability Score (0-100)\nReflects the quality and quantity of the node's connections.\nCombines two sub-factors: a) The average Signal-to-Noise Ratio (SNR) of the node's connections. b) The number of neighbors the node has.\nA higher score suggests more reliable and robust connections.", + "description": "Total packets sent in specified time range", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [ - { - "options": { - "importance_score": { - "index": 0, - "text": "Importance Score" - } - }, - "type": "value" - } - ], + "fieldMinMax": false, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "Packets" }, "overrides": [] }, @@ -61,10 +57,10 @@ "x": 0, "y": 0 }, - "id": 13, + "id": 10, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -83,51 +79,21 @@ "targets": [ { "datasource": { - "type": "postgres", - "uid": "PA942B37CCFAF5A81" + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "WITH RECURSIVE\n-- Find all paths in the network\npaths(start_node, end_node, path, depth) AS (\n SELECT node_id, neighbor_id, ARRAY[node_id, neighbor_id], 1\n FROM node_neighbors\n UNION ALL\n SELECT p.start_node, nn.neighbor_id, p.path || nn.neighbor_id, p.depth + 1\n FROM paths p\n JOIN node_neighbors nn ON p.end_node = nn.node_id\n WHERE nn.neighbor_id <> ALL(p.path) AND p.depth < 10 -- Limit path depth to avoid cycles\n),\n-- Calculate node importance based on how many shortest paths it's on\nnode_importance AS (\n SELECT node_id, COUNT(*) as path_count\n FROM (\n SELECT DISTINCT ON (start_node, unnest) unnest AS node_id\n FROM (\n SELECT start_node, end_node, unnest(path[2:array_length(path,1)-1])\n FROM (\n SELECT DISTINCT ON (start_node, end_node) *\n FROM paths\n ORDER BY start_node, end_node, depth\n ) shortest_paths\n WHERE array_length(path, 1) > 2\n ) exploded_paths\n ) nodes_on_paths\n GROUP BY node_id\n),\n-- Calculate average SNR and neighbor count for stability score\nnode_stability AS (\n SELECT\n node_id,\n AVG(snr) as avg_snr,\n COUNT(*) as neighbor_count\n FROM node_neighbors\n GROUP BY node_id\n),\n-- Calculate the maximum path_count and neighbor_count for normalization\nmax_values AS (\n SELECT \n MAX(ni.path_count) as max_path_count,\n MAX(ns.neighbor_count) as max_neighbor_count\n FROM node_importance ni\n CROSS JOIN node_stability ns\n)\nSELECT\n cd.node_id,\n cd.short_name,\n cd.role,\n -- Revised Importance Score (0-100)\n -- Combine normalized path_count and neighbor_count using a logarithmic scale\n LEAST(\n (LN(COALESCE(ni.path_count, 1)) / LN(GREATEST(mv.max_path_count, 2))) * 50 +\n (LN(COALESCE(ns.neighbor_count, 1)) / LN(GREATEST(mv.max_neighbor_count, 2))) * 50,\n 100\n )::INT AS importance_score,\n -- Stability Score (0-100)\n -- Combine SNR quality and neighbor count, normalize to 0-100 scale\n LEAST(\n (COALESCE(ns.avg_snr, 0) + 20) * 2.5 + -- SNR typically ranges from -20 to 20, so we add 20 and multiply by 2.5 to get 0-100\n LEAST(COALESCE(ns.neighbor_count, 0) * 10, 50), -- Up to 50 points for number of neighbors\n 100\n )::INT AS stability_score\nFROM\n client_details cd\nLEFT JOIN node_importance ni ON cd.node_id = ni.node_id\nLEFT JOIN node_stability ns ON cd.node_id = ns.node_id\nCROSS JOIN max_values mv\nWHERE\n cd.node_id = ${nodeID:singlequote}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(sum by(portnum) (mesh_packet_source_types_total{source_id=~\"$nodeID\"}))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Mesh Node Scoring System", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "importance_score": "Importance Score", - "node_id": "Node ID", - "role": "Client Role", - "short_name": "Short Name", - "stability_score": "Stability Score" - } - } + "useBackend": false } ], + "title": "Total packets sent", "type": "stat" }, { @@ -157,7 +123,7 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 4, "w": 13, "x": 5, "y": 0 @@ -269,81 +235,6 @@ "title": "Last message received", "type": "stat" }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Total packets sent in specified time range", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Packets" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 4 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(sum by(portnum) (mesh_packet_source_types_total{source_id=~\"$nodeID\"}))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Total packets sent", - "type": "stat" - }, { "datasource": { "type": "postgres", @@ -399,7 +290,7 @@ "h": 4, "w": 24, "x": 0, - "y": 8 + "y": 4 }, "id": 1, "options": { @@ -424,7 +315,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT * FROM client_details WHERE node_id = '${nodeID}' LIMIT 1 ", + "rawSql": "SELECT * FROM node_details WHERE node_id = '${nodeID}' LIMIT 1 ", "refId": "A", "sql": { "columns": [ @@ -473,7 +364,7 @@ }, "whereString": "node_id = '${nodeID}'" }, - "table": "client_details" + "table": "node_details" } ], "title": "Node details", @@ -498,250 +389,6 @@ "transparent": true, "type": "table" }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 20 - }, - { - "color": "green", - "value": 40 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 12 - }, - "id": 2, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "telemetry_app_battery_level{node_id=\"$nodeID\"}", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Battery Level", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "volt" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 12 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "telemetry_app_voltage{node_id=\"$nodeID\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Device voltage", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 16, - "x": 8, - "y": 12 - }, - "id": 5, - "options": { - "alignValue": "center", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "topk(1, telemetry_app_temperature{node_id=\"$nodeID\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{long_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Temprerature telemetry", - "transformations": [ - { - "id": "seriesToRows", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": {} - } - } - ], - "type": "state-timeline" - }, { "datasource": { "type": "postgres", @@ -749,10 +396,10 @@ }, "description": "Graph that is built from Neighbor Info reports and shows the signal strenth for each line", "gridPos": { - "h": 10, + "h": 8, "w": 8, "x": 0, - "y": 17 + "y": 8 }, "id": 9, "options": { @@ -773,7 +420,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n node_id AS \"id\", \n long_name AS \"title\", \n hardware_model AS \"detail__Hardware Detail\", \n role AS \"detail__Client Role\", \n mqtt_status AS \"detail__MQTT Status\", \n short_name AS \"subtitle\",\n CASE \n WHEN mqtt_status = 'online' THEN '#2ECC71' -- Green for online\n WHEN mqtt_status = 'offline' THEN '#E74C3C' -- Red for offline\n ELSE '#808080' -- Gray for none or any other status\n END AS \"color\"\nFROM \n client_details\n WHERE node_id IN (${nodeID:singlequote}) OR node_id IN (${relatives:singlequote})", + "rawSql": "SELECT \n node_id AS \"id\", \n long_name AS \"title\", \n hardware_model AS \"detail__Hardware Detail\", \n role AS \"detail__Client Role\", \n mqtt_status AS \"detail__MQTT Status\", \n short_name AS \"subtitle\",\n CASE \n WHEN mqtt_status = 'online' THEN '#2ECC71' -- Green for online\n WHEN mqtt_status = 'offline' THEN '#E74C3C' -- Red for offline\n ELSE '#808080' -- Gray for none or any other status\n END AS \"color\"\nFROM \n node_details\n WHERE node_id IN (${nodeID:singlequote}) OR node_id IN (${relatives:singlequote})", "refId": "nodes", "sql": { "columns": [ @@ -848,7 +495,7 @@ ], "limit": 50 }, - "table": "client_details" + "table": "node_details" }, { "datasource": { @@ -919,6 +566,102 @@ "title": "Node Graph", "type": "nodeGraph" }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "topk(1, telemetry_app_temperature{node_id=\"$nodeID\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{long_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Temprerature telemetry", + "transformations": [ + { + "id": "seriesToRows", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Metric": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "state-timeline" + }, { "datasource": { "type": "prometheus", @@ -958,7 +701,7 @@ "h": 5, "w": 16, "x": 8, - "y": 17 + "y": 13 }, "id": 6, "options": { @@ -1015,6 +758,193 @@ ], "type": "state-timeline" }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PA942B37CCFAF5A81" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "tooltip": true, + "type": "osm-standard" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": true, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": false, + "style": { + "color": { + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "latitude": "latitude_norm", + "longitude": "longitude_norm", + "mode": "coords" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "none" + }, + "view": { + "allLayers": true, + "id": "fit", + "lastOnly": false, + "lat": 0, + "layer": "Layer 1", + "lon": 0, + "zoom": 15 + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PA942B37CCFAF5A81" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT * ,\nlongitude * 1e-7 as \"longitude_norm\",\nlatitude * 1e-7 as \"latitude_norm\"\nFROM node_details \nWHERE node_id = '${nodeID}' LIMIT 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "*", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "abb9a8b8-4567-489a-bcde-f190ef655acf", + "properties": { + "field": "node_id", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "${nodeID}" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + } + ], + "id": "aabba98a-0123-4456-b89a-b190ef5f79f0", + "properties": { + "conjunction": "AND" + }, + "type": "group" + }, + "whereString": "node_id = '${nodeID}'" + }, + "table": "node_details" + } + ], + "title": "Panel Title", + "type": "geomap" + }, { "datasource": { "type": "prometheus", @@ -1054,7 +984,7 @@ "h": 5, "w": 16, "x": 8, - "y": 22 + "y": 18 }, "id": 7, "options": { @@ -1110,6 +1040,210 @@ } ], "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 110, + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "telemetry_app_battery_level{node_id=\"$nodeID\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{long_name}} | Model: {{hardware_model}} | Role: {{role}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Battery Level", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "volt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "telemetry_app_voltage{node_id=\"$nodeID\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{long_name}} | Model: {{hardware_model}} | Role: {{role}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Device voltage", + "type": "timeseries" } ], "schemaVersion": 39, @@ -1119,21 +1253,21 @@ { "current": { "selected": false, - "text": "🐦‍🔥:34GL3 (3944975137)", - "value": "3944975137" + "text": "🤫 (3663649648)", + "value": "3663649648" }, "datasource": { "type": "postgres", "uid": "PA942B37CCFAF5A81" }, - "definition": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM client_details \nORDER BY long_name", + "definition": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM node_details \nORDER BY long_name", "hide": 0, "includeAll": false, "label": "Node ID", "multi": false, "name": "nodeID", "options": [], - "query": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM client_details \nORDER BY long_name", + "query": "SELECT \n concat(long_name, ' (', node_id, ')') as __text, \n node_id as __value \nFROM node_details \nORDER BY long_name", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1142,9 +1276,10 @@ }, { "current": { + "isNone": true, "selected": false, - "text": "1129710492", - "value": "1129710492" + "text": "None", + "value": "" }, "datasource": { "type": "postgres", @@ -1175,6 +1310,6 @@ "timezone": "browser", "title": "Node Dashboard", "uid": "edqo1uh0eglq8g", - "version": 2, + "version": 4, "weekStart": "" } \ No newline at end of file diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql index 2ae31b6..1e8f37a 100644 --- a/docker/postgres/init.sql +++ b/docker/postgres/init.sql @@ -1,3 +1,5 @@ +CREATE EXTENSION IF NOT EXISTS moddatetime; + CREATE TABLE IF NOT EXISTS messages ( id TEXT PRIMARY KEY, @@ -29,13 +31,10 @@ CREATE TABLE IF NOT EXISTS node_details role VARCHAR, mqtt_status VARCHAR default 'none', -- Location Data - longitude FLOAT, - latitude FLOAT, - altitude FLOAT, - precision FLOAT, - country VARCHAR, - city VARCHAR, - state VARCHAR, + longitude INT, + latitude INT, + altitude INT, + precision INT, -- SQL Data created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL diff --git a/exporter/processor_base.py b/exporter/processor_base.py index 80eaed8..a775aeb 100644 --- a/exporter/processor_base.py +++ b/exporter/processor_base.py @@ -12,7 +12,7 @@ except ImportError: from meshtastic.protobuf.mesh_pb2 import MeshPacket, Data, HardwareModel from meshtastic.protobuf.portnums_pb2 import PortNum -from prometheus_client import CollectorRegistry, Counter, Histogram, Gauge +from prometheus_client import CollectorRegistry, Counter, Gauge from psycopg_pool import ConnectionPool from exporter.client_details import ClientDetails @@ -46,10 +46,14 @@ class MessageProcessor: 'destination_role' ] - self.message_size_in_bytes = Histogram( + reduced_labels = [ + 'source_id', 'destination_id' + ] + + self.message_size_in_bytes = Gauge( 'text_message_app_size_in_bytes', 'Size of text messages processed by the app in Bytes', - common_labels + ['portnum'], + reduced_labels + ['portnum'], registry=self.registry ) @@ -74,7 +78,7 @@ class MessageProcessor: registry=self.registry ) # Histogram for the rx_time (time in seconds) - self.rx_time_histogram = Histogram( + self.rx_time_histogram = Gauge( 'mesh_packet_rx_time', 'Receive time of mesh packets (seconds since 1970)', common_labels, @@ -205,10 +209,15 @@ class MessageProcessor: 'destination_role': destination_client_details.role, } + reduced_labels = { + 'source_id': source_client_details.node_id, + 'destination_id': destination_client_details.node_id + } + self.message_size_in_bytes.labels( - **common_labels, + **reduced_labels, portnum=self.get_port_name_from_portnum(port_num) - ).observe(sys.getsizeof(mesh_packet)) + ).set(sys.getsizeof(mesh_packet)) self.source_message_type_counter.labels( **common_labels, @@ -226,7 +235,7 @@ class MessageProcessor: self.rx_time_histogram.labels( **common_labels - ).observe(mesh_packet.rx_time) + ).set(mesh_packet.rx_time) self.rx_snr_gauge.labels( **common_labels diff --git a/exporter/processors.py b/exporter/processors.py index eacda36..66f94c1 100644 --- a/exporter/processors.py +++ b/exporter/processors.py @@ -167,7 +167,7 @@ class NodeInfoAppProcessor(Processor): conn.commit() - self.metrics.db.execute_db_operation(db_operation) + self.metrics.get_db().execute_db_operation(db_operation) @ProcessorRegistry.register_processor(PortNum.ROUTING_APP) @@ -499,24 +499,8 @@ class NeighborInfoAppProcessor(Processor): except Exception as e: logger.error(f"Failed to parse NEIGHBORINFO_APP packet: {e}") return - self.update_node_graph(neighbor_info, client_details) self.update_node_neighbors(neighbor_info, client_details) - def update_node_graph(self, neighbor_info: NeighborInfo, client_details: ClientDetails): - def operation(cur, conn): - cur.execute(""" - INSERT INTO node_graph (node_id, last_sent_by_node_id, broadcast_interval_secs) - VALUES (%s, %s, %s) - ON CONFLICT (node_id) - DO UPDATE SET - last_sent_by_node_id = EXCLUDED.last_sent_by_node_id, - broadcast_interval_secs = EXCLUDED.broadcast_interval_secs, - last_sent_at = CURRENT_TIMESTAMP - """, (client_details.node_id, neighbor_info.last_sent_by_id, neighbor_info.node_broadcast_interval_secs)) - conn.commit() - - self.metrics.db.execute_db_operation(operation) - def update_node_neighbors(self, neighbor_info: NeighborInfo, client_details: ClientDetails): def operation(cur, conn): new_neighbor_ids = [str(neighbor.node_id) for neighbor in neighbor_info.neighbors] @@ -549,7 +533,7 @@ class NeighborInfoAppProcessor(Processor): conn.commit() - self.metrics.db.execute_db_operation(operation) + self.metrics.get_db().execute_db_operation(operation) @ProcessorRegistry.register_processor(PortNum.ATAK_PLUGIN) diff --git a/exporter/registry.py b/exporter/registry.py index ec53d56..afd126a 100644 --- a/exporter/registry.py +++ b/exporter/registry.py @@ -1,4 +1,5 @@ -import geopy.point +import uuid + from geopy.geocoders import Nominatim from prometheus_client import CollectorRegistry, Counter, Gauge @@ -19,8 +20,11 @@ class _Metrics: self._registry = registry self._init_metrics() self.initialized = True # Attribute to indicate initialization - self.geolocator = Nominatim() - self.db = db + self.geolocator = Nominatim(user_agent=f"meshtastic-prometheus-exporter-{str(uuid.uuid4())}") + self.db = db + + def get_db(self): + return self.db @staticmethod def _get_common_labels(): @@ -36,22 +40,24 @@ class _Metrics: self._init_route_discovery_metrics() def update_metrics_position(self, latitude, longitude, altitude, precision, client_details: ClientDetails): - point = geopy.point.Point(latitude, longitude, altitude) - location = self.geolocator.reverse(point, language='en') + # Could be used to calculate more complex data (Like distances etc..) + # point = geopy.point.Point(latitude, longitude, altitude) # Not used for now - country = location.raw.get('address', {}).get('country', 'Unknown') - city = location.raw.get('address', {}).get('city', 'Unknown') - state = location.raw.get('address', {}).get('state', 'Unknown') + if latitude != 0 and longitude != 0: + # location = RateLimiter(self.geolocator.reverse, min_delay_seconds=10, swallow_exceptions=False)((latitude, longitude), language='en', timeout=10) + # country = location.raw.get('address', {}).get('country', 'Unknown') + # city = location.raw.get('address', {}).get('city', 'Unknown') + # state = location.raw.get('address', {}).get('state', 'Unknown') - def db_operation(cur, conn): - cur.execute(""" - UPDATE node_details - SET latitude = %s, longitude = %s, altitude = %s, precision = %s, country = %s, city = %s, state = %s - WHERE node_id = %s - """, (latitude, longitude, altitude, precision, country, city, state, client_details.node_id)) - conn.commit() + def db_operation(cur, conn): + cur.execute(""" + UPDATE node_details + SET latitude = %s, longitude = %s, altitude = %s, precision = %s + WHERE node_id = %s + """, (latitude, longitude, altitude, precision, client_details.node_id)) + conn.commit() - self.db.execute_db_operation(db_operation) + self.db.execute_db_operation(db_operation) def _init_metrics_telemetry_power(self): self.ch1_voltage_gauge = Gauge( diff --git a/requirements.txt b/requirements.txt index bfe29f9..c19a5e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ cryptography~=42.0.8 psycopg~=3.1.19 psycopg_pool~=3.2.2 meshtastic~=2.3.13 -psycopg-binary~=3.1.20 \ No newline at end of file +psycopg-binary~=3.1.20 +geopy>=2.4.1 \ No newline at end of file