DRCIT/meshtastic-metrics-exporter
1
0
Fork 0
forked from mudhorn/meshtastic-metrics-exporter
Code Pull requests Activity

Merge pull request #57 from tcivie/node-configuration-metrics

Browse source 
Node configuration metrics
This commit is contained in:
Gleb Tcivie 2024-08-09 13:41:18 +03:00 committed by GitHub
parent 36ce36287e 6dc3116622
commit 94d8512818
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 1870 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env
View file

@ -35,3 +35,6 @@ MQTT_SERVER_KEY=1PG7OiApB1nwvP+rz05pAQ==
# Message types to filter (default: none) (comma separated) (eg. TEXT_MESSAGE_APP,POSITION_APP)
# Full list can be found here: https://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum
EXPORTER_MESSAGE_TYPES_TO_FILTER=TEXT_MESSAGE_APP
# Enable node configurations report (default: true)
REPORT_NODE_CONFIGURATIONS=true

2
.idea/sqldialects.xml
View file

@ -2,7 +2,7 @@
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/docker/postgres/init.sql" dialect="PostgreSQL" />
<file url="file://$PROJECT_DIR$/exporter/processor_base.py" dialect="PostgreSQL" />
<file url="file://$PROJECT_DIR$/exporter/processor/processor_base.py" dialect="PostgreSQL" />
<file url="PROJECT" dialect="PostgreSQL" />
</component>
</project>

3
docker-compose.yml
View file

@ -35,6 +35,9 @@ services:
context: .
dockerfile: ./docker/exporter/Dockerfile.exporter
restart: unless-stopped
depends_on:
- prometheus
- postgres
extra_hosts:
- "host.docker.internal:host-gateway"
env_file:

424
docker/grafana/provisioning/dashboards/Investigation Board.json Normal file
View file

@ -0,0 +1,424 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 13,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
"description": "Information about the clients we have in the network and their relative packets sent",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "center",
"cellOptions": {
"type": "auto",
"wrapText": true
},
"filterable": true,
"inspect": true,
"minWidth": 180
},
"fieldMinMax": true,
"mappings": [
{
"options": {
"none": {
"color": "text",
"index": 2,
"text": "⚪️ Unknown"
},
"offline": {
"color": "red",
"index": 1,
"text": "🛑 offline"
},
"online": {
"color": "green",
"index": 0,
"text": "🟢 online"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "#EAB839",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "MAP_REPORT_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"drawStyle": "line",
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "NEIGHBORINFO_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "NODEINFO_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"drawStyle": "line",
"hideValue": false,
"lineStyle": {
"dash": [
10,
10
],
"fill": "solid"
},
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "POSITION_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "RANGE_TEST_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "ROUTING_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "TELEMETRY_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "TEXT_MESSAGE_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "TRACEROUTE_APP"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"hideValue": false,
"type": "sparkline"
}
},
{
"id": "unit",
"value": "packets"
}
]
},
{
"matcher": {
"id": "byName",
"options": "node_id"
},
"properties": [
{
"id": "unit",
"value": "hex"
}
]
}
]
},
"gridPos": {
"h": 18,
"w": 24,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"cellHeight": "md",
"footer": {
"countRows": true,
"enablePagination": true,
"fields": [],
"reducer": [
"count"
],
"show": true
},
"frameIndex": 1,
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "TELEMETRY_APP"
}
]
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT * FROM node_details",
"refId": "Client Details",
"sql": {
"columns": [
{
"parameters": [
{
"name": "*",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "client_details"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(portnum, source_id) (mesh_packet_destination_types_total)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"legendFormat": "__auto",
"range": true,
"refId": "Packet Types",
"useBackend": false
}
],
"title": "General Information",
"transformations": [
{
"filter": {
"id": "byRefId",
"options": "Packet Types"
},
"id": "timeSeriesTable",
"options": {
"Packet Types": {
"stat": "lastNotNull",
"timeField": "Time"
}
}
},
{
"filter": {
"id": "byRefId",
"options": "Packet Types"
},
"id": "groupingToMatrix",
"options": {
"columnField": "portnum",
"emptyValue": "null",
"rowField": "source_id",
"valueField": "Trend #Packet Types"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "(source_id\\\\portnum)",
"renamePattern": "node_id"
}
},
{
"id": "joinByField",
"options": {
"byField": "node_id",
"mode": "outer"
}
}
],
"transparent": true,
"type": "table"
}
],
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Investigation Board",
"uid": "adrqynul4j3eoa",
"version": 17,
"weekStart": ""
}

560
docker/grafana/provisioning/dashboards/Main Dashboard.json
View file

@ -44,7 +44,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"id": 11,
"links": [],
"panels": [
{
@ -175,10 +175,6 @@
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
@ -187,7 +183,7 @@
},
"gridPos": {
"h": 4,
"w": 5,
"w": 4,
"x": 4,
"y": 0
},
@ -231,6 +227,114 @@
"title": "Active nodes in the last 30 minutes",
"type": "stat"
},
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"description": "How many nodes are Unknonw and how many are Known.\n\nThe unknown are ones that are either NULL in their long name or \"Unknown\" in their value",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "nodes"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Unnamed"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 0
},
"id": 24,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT\n COUNT(CASE WHEN long_name IS NULL OR long_name = 'Unknown' AND node_id != '0' THEN 1 END) AS \"Unnamed\",\n COUNT(CASE WHEN long_name IS NOT NULL AND long_name != 'Unknown' AND node_id != '0' THEN 1 END) AS \"Named\"\nFROM\n node_details\n\n",
"refId": "A",
"sql": {
"columns": [
{
"name": "COUNT",
"parameters": [
{
"name": "short_name",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"name": "short_name",
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "node_details"
}
],
"title": "Nodes naming status",
"type": "stat"
},
{
"datasource": {
"type": "postgres",
@ -279,8 +383,8 @@
},
"gridPos": {
"h": 4,
"w": 5,
"x": 9,
"w": 4,
"x": 12,
"y": 0
},
"id": 17,
@ -298,7 +402,7 @@
"values": true
},
"showPercentChange": false,
"textMode": "auto",
"textMode": "value_and_name",
"wideLayout": true
},
"pluginVersion": "11.1.0",
@ -394,78 +498,6 @@
"title": "MQTT Node status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "This metric shows the length that only 5% of messages exceed, representing the typical upper limit of message sizes in our Meshtastic network. It helps identify trends in message length without being skewed by rare, extremely long outliers. By focusing on the 95th percentile, we get a reliable measure of 'long' messages while filtering out unusual spikes.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 0
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "histogram_quantile(0.95, sum by(le) (rate(text_message_app_size_in_bytes_bucket{source_id=~\"$Nodes\"}[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Typical Maximum Message Length in Bytes",
"type": "stat"
},
{
"datasource": {
"type": "datasource",
@ -498,8 +530,8 @@
},
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"w": 4,
"x": 16,
"y": 0
},
"id": 22,
@ -541,13 +573,209 @@
"title": "Total packets sent",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "Shows the total data sent on the mesh in bytes in the last hour. This includes all types of packets.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 20,
"y": 0
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum(sum_over_time(text_message_app_size_in_bytes{source_id=~\"$Nodes\"}[1h]))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Data sent on mesh in last hour",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "Displays the nodes that utilize most of the channel (On average with at least 1 update every hour)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "dashed+area"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 8.5
},
{
"color": "red",
"value": 10
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 0,
"y": 4
},
"id": 26,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "topk(3,\n avg by (node_id, short_name, long_name) (\n avg_over_time(telemetry_app_channel_utilization{node_id=~\"$Nodes\"}[$__range]) > 0\n and\n timestamp(telemetry_app_channel_utilization{node_id=~\"$Nodes\"}) > time() - 3600\n )\n)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{short_name}}:{{long_name}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Top channel utilizers",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 4
"y": 13
},
"id": 25,
"panels": [],
"title": "Highlighted Nodes",
"type": "row"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 16,
"panels": [],
@ -615,11 +843,11 @@
},
{
"color": "yellow",
"value": 10
"value": 8.5
},
{
"color": "red",
"value": 20
"value": 10
}
]
},
@ -631,7 +859,7 @@
"h": 9,
"w": 11,
"x": 0,
"y": 5
"y": 15
},
"id": 23,
"options": {
@ -651,23 +879,6 @@
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "max(telemetry_app_channel_utilization{node_id=~\"$Nodes\"})",
"fullMetaSearch": false,
"hide": true,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Chanel utilization (Max)",
"range": true,
"refId": "Max Chanel Utilization recorded",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
@ -706,7 +917,7 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 6.5,
"axisSoftMax": 10,
"barAlignment": 0,
"drawStyle": "points",
"fillOpacity": 0,
@ -760,7 +971,7 @@
"h": 9,
"w": 13,
"x": 11,
"y": 5
"y": 15
},
"id": 3,
"options": {
@ -813,7 +1024,7 @@
"h": 32,
"w": 11,
"x": 0,
"y": 14
"y": 24
},
"id": 20,
"options": {
@ -1009,7 +1220,7 @@
"h": 10,
"w": 5,
"x": 11,
"y": 14
"y": 24
},
"id": 5,
"options": {
@ -1075,16 +1286,23 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "#EAB839",
"value": 4
},
{
"color": "semi-dark-orange",
"value": 5
},
{
"color": "red",
"value": 6
},
{
"color": "dark-red",
"value": 7
}
]
},
@ -1096,7 +1314,7 @@
"h": 10,
"w": 8,
"x": 16,
"y": 14
"y": 24
},
"id": 9,
"options": {
@ -1138,7 +1356,7 @@
},
{
"datasource": {
"type": "postgres",
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"description": "Information stored on the Redis DB which includes \"hard to get\" information - that is needed to be gathered over time.",
@ -1232,7 +1450,7 @@
"properties": [
{
"id": "unit",
"value": "dateTimeAsLocal"
"value": "dateTimeFromNow"
}
]
},
@ -1247,6 +1465,30 @@
"value": 126
}
]
},
{
"matcher": {
"id": "byName",
"options": "Last Updated At"
},
"properties": [
{
"id": "unit",
"value": "dateTimeFromNow"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Node ID Hex"
},
"properties": [
{
"id": "unit",
"value": "hex"
}
]
}
]
},
@ -1254,7 +1496,7 @@
"h": 22,
"w": 13,
"x": 11,
"y": 24
"y": 34
},
"id": 14,
"options": {
@ -1268,11 +1510,12 @@
],
"show": true
},
"frameIndex": 0,
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "MQTT Status"
"desc": false,
"displayName": "Long name"
}
]
},
@ -1286,7 +1529,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT * FROM node_details WHERE node_id <> '0'",
"rawSql": "SELECT * FROM node_details WHERE node_id <> '0' ",
"refId": "A",
"sql": {
"columns": [
@ -1349,12 +1592,44 @@
"source": "Value"
}
},
{
"id": "calculateField",
"options": {
"alias": "Node ID Hex",
"mode": "reduceRow",
"reduce": {
"include": [
"node_id"
],
"reducer": "lastNotNull"
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"excludeByName": {
"altitude": true,
"latitude": true,
"longitude": true,
"precision": true
},
"includeByName": {},
"indexByName": {},
"indexByName": {
"Node ID Hex": 1,
"altitude": 9,
"created_at": 11,
"hardware_model": 4,
"latitude": 8,
"long_name": 3,
"longitude": 7,
"mqtt_status": 6,
"node_id": 0,
"precision": 10,
"role": 5,
"short_name": 2,
"updated_at": 12
},
"renameByName": {
"created_at": "Record Created At",
"hardware_model": "Hardware Model",
@ -1362,7 +1637,8 @@
"mqtt_status": "MQTT Status",
"node_id": "Node ID",
"role": "Client Role",
"short_name": "Short Name"
"short_name": "Short Name",
"updated_at": "Last Updated At"
}
}
}
@ -1419,13 +1695,38 @@
},
"unit": "string"
},
"overrides": []
"overrides": [
{
"matcher": {
"id": "byName",
"options": "created_at"
},
"properties": [
{
"id": "unit",
"value": "dateTimeFromNow"
}
]
},
{
"matcher": {
"id": "byName",
"options": "updated_at"
},
"properties": [
{
"id": "unit",
"value": "dateTimeFromNow"
}
]
}
]
},
"gridPos": {
"h": 25,
"w": 24,
"x": 0,
"y": 46
"y": 56
},
"id": 8,
"options": {
@ -1453,7 +1754,7 @@
"field": "mqtt_status",
"fixed": "dark-green"
},
"opacity": 0.2,
"opacity": 1,
"rotation": {
"fixed": 0,
"max": 360,
@ -1461,12 +1762,12 @@
"mode": "mod"
},
"size": {
"fixed": 10,
"fixed": 3,
"max": 5,
"min": 1
},
"symbol": {
"fixed": "img/icons/marker/triangle.svg",
"fixed": "img/icons/marker/circle.svg",
"mode": "fixed"
},
"symbolAlign": {
@ -1474,7 +1775,6 @@
"vertical": "top"
},
"text": {
"field": "Short Name",
"fixed": "",
"mode": "field"
},
@ -1593,11 +1893,15 @@
"renameByName": {
"Time 1": "Log Time",
"__name__ 1": "",
"hardware_model": "Hardware Model",
"hardware_model 1": "Hardware Model",
"long_name": "Long Name",
"long_name 1": "Long Name",
"mqtt_status": "MQTT_STATUS",
"node_id": "Node ID",
"role": "Client Role",
"role 1": "Client Role",
"short_name": "Short Name",
"short_name 1": "Short Name"
}
}
@ -1673,13 +1977,13 @@
]
},
"time": {
"from": "now-1h",
"from": "now-12h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Main Dashboard",
"uid": "edqkge9mf7v28g",
"version": 5,
"version": 66,
"weekStart": "sunday"
}

317
docker/grafana/provisioning/dashboards/Node Dashboard.json
View file

@ -18,7 +18,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"id": 10,
"links": [],
"panels": [
{
@ -26,7 +26,7 @@
"type": "datasource",
"uid": "-- Mixed --"
},
"description": "Total packets sent in specified time range",
"description": "Total packets sent in the specified time range",
"fieldConfig": {
"defaults": {
"color": {
@ -101,7 +101,7 @@
"type": "datasource",
"uid": "-- Mixed --"
},
"description": "Displays types of packets in the selected time range.\n\nFor more details see:\nhttps://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum",
"description": "Displays types of packets sent in the selected time range.\n\nFor more details see:\nhttps://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum",
"fieldConfig": {
"defaults": {
"color": {
@ -143,7 +143,7 @@
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"textMode": "value_and_name",
"wideLayout": true
},
"pluginVersion": "11.1.0",
@ -164,7 +164,7 @@
"useBackend": false
}
],
"title": "Packet types for selected time",
"title": "Packet types sent for selected time",
"type": "stat"
},
{
@ -172,7 +172,7 @@
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "The last time we got message from this node",
"description": "The last time a packet was observed being sent from this node",
"fieldConfig": {
"defaults": {
"color": {
@ -225,7 +225,221 @@
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "timestamp((\n count(mesh_packet_ids_created{source_id=\"$nodeID\"}) by (source_id)\n -\n count(mesh_packet_ids_created{source_id=\"$nodeID\"} offset 1m) by (source_id)\n) > 0)",
"expr": "timestamp((\n count(mesh_packet_total{source_id=\"$nodeID\"}) by (source_id)\n -\n count(mesh_packet_total{source_id=\"$nodeID\"} offset 1m) by (source_id)\n) > 0)",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Last packet sent",
"type": "stat"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
"description": "Total packets observed with this node as their destination in the specified time range",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "Packets"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 0,
"y": 4
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum(sum by(portnum) (mesh_packet_source_types_total{destination_id=~\"$nodeID\"}))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Total packets received",
"type": "stat"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
"description": "Displays types of packets received in the selected time range.\n\nFor more details see:\nhttps://buf.build/meshtastic/protobufs/docs/main:meshtastic#meshtastic.PortNum",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Packets"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 13,
"x": 5,
"y": 4
},
"id": 15,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "value_and_name",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(portnum) (mesh_packet_source_types_total{destination_id=\"$nodeID\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Packet types received for selected time",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "The last time a packet was observed with this node as its destination",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "dateTimeFromNow"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 4
},
"id": 16,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/^Time$/",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "timestamp((\n count(mesh_packet_total{destination_id=\"$nodeID\"}) by (destination_id)\n -\n count(mesh_packet_total{destination_id=\"$nodeID\"} offset 1m) by (destination_id)\n) > 0)",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -237,7 +451,7 @@
},
{
"datasource": {
"type": "postgres",
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"fieldConfig": {
@ -284,13 +498,50 @@
]
}
},
"overrides": []
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Node Created At"
},
"properties": [
{
"id": "unit",
"value": "dateTimeFromNow"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Node Updated At"
},
"properties": [
{
"id": "unit",
"value": "dateTimeFromNow"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Node ID"
},
"properties": [
{
"id": "unit",
"value": "hex"
}
]
}
]
},
"gridPos": {
"h": 4,
"w": 24,
"x": 0,
"y": 4
"y": 8
},
"id": 1,
"options": {
@ -372,16 +623,23 @@
{
"id": "organize",
"options": {
"excludeByName": {},
"excludeByName": {
"altitude": true,
"latitude": true,
"longitude": true,
"precision": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"created_at": "Node Created At",
"hardware_model": "Hardware Model",
"long_name": "Long Name",
"mqtt_status": "MQTT Status",
"node_id": "Node ID",
"role": "Client Role",
"short_name": "Short Name"
"short_name": "Short Name",
"updated_at": "Node Updated At"
}
}
}
@ -399,7 +657,7 @@
"h": 8,
"w": 8,
"x": 0,
"y": 8
"y": 12
},
"id": 9,
"options": {
@ -605,7 +863,7 @@
"h": 5,
"w": 16,
"x": 8,
"y": 8
"y": 12
},
"id": 5,
"options": {
@ -688,8 +946,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
}
]
},
@ -701,7 +958,7 @@
"h": 5,
"w": 16,
"x": 8,
"y": 13
"y": 17
},
"id": 6,
"options": {
@ -780,8 +1037,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
}
]
}
@ -792,7 +1048,7 @@
"h": 7,
"w": 8,
"x": 0,
"y": 16
"y": 20
},
"id": 13,
"options": {
@ -971,8 +1227,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
}
]
},
@ -984,7 +1239,7 @@
"h": 5,
"w": 16,
"x": 8,
"y": 18
"y": 22
},
"id": 7,
"options": {
@ -1110,7 +1365,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 27
},
"id": 2,
"options": {
@ -1208,7 +1463,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 27
},
"id": 3,
"options": {
@ -1246,6 +1501,7 @@
"type": "timeseries"
}
],
"refresh": "1m",
"schemaVersion": 39,
"tags": [],
"templating": {
@ -1253,8 +1509,8 @@
{
"current": {
"selected": false,
"text": "🤫 (3663649648)",
"value": "3663649648"
"text": "🐦‍🔥:34GL3 (3944975137)",
"value": "3944975137"
},
"datasource": {
"type": "postgres",
@ -1276,10 +1532,9 @@
},
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
"text": "3944975137",
"value": "3944975137"
},
"datasource": {
"type": "postgres",
@ -1310,6 +1565,6 @@
"timezone": "browser",
"title": "Node Dashboard",
"uid": "edqo1uh0eglq8g",
"version": 4,
"version": 30,
"weekStart": ""
}

357
docker/grafana/provisioning/dashboards/Node Graph - Map.json Normal file
View file

@ -0,0 +1,357 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 17,
"links": [],
"panels": [
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PA942B37CCFAF5A81"
},
"description": "Graph that is built from Neighbor Info reports and shows the signal strenth for each line",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "#EAB839",
"value": -13
},
{
"color": "green",
"value": -7
}
]
},
"unit": "dB"
},
"overrides": []
},
"gridPos": {
"h": 24,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"basemap": {
"config": {},
"name": "Layer 0",
"opacity": 1,
"tooltip": true,
"type": "xyz"
},
"controls": {
"mouseWheelZoom": true,
"showAttribution": true,
"showDebug": false,
"showMeasure": true,
"showScale": true,
"showZoom": true
},
"layers": [
{
"config": {
"arrow": 1,
"edgeStyle": {
"color": {
"field": "mainstat",
"fixed": "dark-green"
},
"opacity": 1,
"rotation": {
"fixed": 0,
"max": 360,
"min": -360,
"mode": "mod"
},
"size": {
"field": "thickness",
"fixed": 5,
"max": 3,
"min": 1
},
"symbol": {
"fixed": "img/icons/marker/circle.svg",
"mode": "fixed"
},
"symbolAlign": {
"horizontal": "center",
"vertical": "center"
},
"text": {
"field": "mainstat",
"fixed": "",
"mode": "field"
},
"textConfig": {
"fontSize": 10,
"offsetX": 0,
"offsetY": 0,
"textAlign": "center",
"textBaseline": "top"
}
},
"showLegend": false,
"style": {
"color": {
"fixed": "dark-green"
},
"opacity": 1,
"rotation": {
"fixed": 5,
"max": 360,
"min": -360,
"mode": "mod"
},
"size": {
"fixed": 5,
"max": 15,
"min": 2
},
"symbol": {
"fixed": "img/icons/marker/circle.svg",
"mode": "fixed"
},
"symbolAlign": {
"horizontal": "center",
"vertical": "center"
},
"text": {
"field": "title",
"fixed": "",
"mode": "field"
},
"textConfig": {
"fontSize": 12,
"offsetX": 0,
"offsetY": 0,
"textAlign": "left",
"textBaseline": "top"
}
}
},
"name": "Layer 1",
"tooltip": true,
"type": "network"
}
],
"tooltip": {
"mode": "none"
},
"view": {
"allLayers": true,
"id": "coords",
"lat": 31.991767,
"lon": 34.703985,
"zoom": 7.65
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "PA942B37CCFAF5A81"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT DISTINCT\n cd.node_id AS \"id\",\n cd.long_name AS \"title\",\n cd.hardware_model AS \"detail__Hardware Detail\",\n cd.role AS \"detail__Client Role\",\n cd.mqtt_status AS \"detail__MQTT Status\",\n cd.short_name AS \"subtitle\",\n cd.longitude * 1e-7 AS \"longitude\",\n cd.latitude * 1e-7 AS \"latitude\"\nFROM\n node_details cd\nLEFT JOIN (\n SELECT node_id FROM node_neighbors\n UNION\n SELECT neighbor_id FROM node_neighbors\n) nn ON cd.node_id = nn.node_id\nWHERE nn.node_id IS NOT NULL",
"refId": "nodes",
"sql": {
"columns": [
{
"alias": "\"id\"",
"parameters": [
{
"name": "node_id",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"title\"",
"parameters": [
{
"name": "long_name",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"detail__Hardware Detail\"",
"parameters": [
{
"name": "hardware_model",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"detail__Client Role\"",
"parameters": [
{
"name": "role",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"detail__MQTT Status\"",
"parameters": [
{
"name": "mqtt_status",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"subtitle\"",
"parameters": [
{
"name": "short_name",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "node_details"
},
{
"datasource": {
"type": "postgres",
"uid": "PA942B37CCFAF5A81"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n CONCAT(neighbor_id, '_', node_id) AS id,\n neighbor_id AS \"source\",\n node_id AS \"target\",\n snr AS \"mainstat\",\n CASE\n WHEN snr < -13 THEN '#E74C3C' -- Red for SNR < -13\n WHEN snr < -7 THEN '#F4D03F' -- Yellow for -13 ≤ SNR < -7\n ELSE '#2ECC71' -- Green for SNR ≥ -7\n END AS \"color\",\n GREATEST(0.1, LEAST(2, 1 + ((snr + 13) / 10))) AS \"thickness\"\nFROM \n node_neighbors",
"refId": "edges",
"sql": {
"columns": [
{
"parameters": [
{
"name": "id",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"source\"",
"parameters": [
{
"name": "neighbor_id",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"target\"",
"parameters": [
{
"name": "node_id",
"type": "functionParameter"
}
],
"type": "function"
},
{
"alias": "\"mainstat\"",
"parameters": [
{
"name": "snr",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "node_neighbors"
}
],
"title": "Node Graph (Map)",
"type": "geomap"
}
],
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Node Graph (Map)",
"uid": "cdt467x8uwbgga",
"version": 5,
"weekStart": ""
}

2
docker/grafana/provisioning/dashboards/dashboard.yml
View file

@ -3,7 +3,7 @@ apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
folder: 'Main Dashboards'
type: file
disableDeletion: false
updateIntervalSeconds: 10

103
docker/postgres/init.sql
View file

@ -1,5 +1,3 @@
CREATE EXTENSION IF NOT EXISTS moddatetime;
CREATE TABLE IF NOT EXISTS messages
(
id TEXT PRIMARY KEY,
@ -53,8 +51,99 @@ CREATE TABLE IF NOT EXISTS node_neighbors
CREATE UNIQUE INDEX idx_unique_node_neighbor ON node_neighbors (node_id, neighbor_id);
CREATE OR REPLACE TRIGGER client_details_updated_at
BEFORE UPDATE
ON node_details
FOR EACH ROW
EXECUTE PROCEDURE moddatetime(updated_at);
CREATE TABLE IF NOT EXISTS node_configurations
(
node_id VARCHAR PRIMARY KEY,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-- Configuration (Telemetry)
environment_update_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
environment_update_last_timestamp TIMESTAMP DEFAULT NOW(),
device_update_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
device_update_last_timestamp TIMESTAMP DEFAULT NOW(),
air_quality_update_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
air_quality_update_last_timestamp TIMESTAMP DEFAULT NOW(),
power_update_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
power_update_last_timestamp TIMESTAMP DEFAULT NOW(),
-- Configuration (Range Test)
range_test_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
range_test_packets_total INT DEFAULT 0, -- in packets
range_test_first_packet_timestamp TIMESTAMP DEFAULT NOW(),
range_test_last_packet_timestamp TIMESTAMP DEFAULT NOW(),
-- Configuration (PAX Counter)
pax_counter_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
pax_counter_last_timestamp TIMESTAMP DEFAULT NOW(),
-- Configuration (Neighbor Info)
neighbor_info_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
neighbor_info_last_timestamp TIMESTAMP DEFAULT NOW(),
-- Configuration (MQTT)
mqtt_encryption_enabled BOOLEAN DEFAULT FALSE,
mqtt_json_enabled BOOLEAN DEFAULT FALSE,
mqtt_configured_root_topic TEXT DEFAULT '',
mqtt_info_last_timestamp TIMESTAMP DEFAULT NOW(),
-- Configuration (Map)
map_broadcast_interval INTERVAL DEFAULT '0 seconds' NOT NULL,
map_broadcast_last_timestamp TIMESTAMP DEFAULT NOW(),
-- FOREIGN KEY (node_id) REFERENCES node_details (node_id),
UNIQUE (node_id)
);
-- -- Function to update old values
-- CREATE OR REPLACE FUNCTION update_old_node_configurations()
-- RETURNS TRIGGER AS $$
-- BEGIN
-- -- Update intervals to 0 if not updated in 24 hours
-- IF NEW.environment_update_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.environment_update_interval := '0 seconds';
-- END IF;
--
-- IF NEW.device_update_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.device_update_interval := '0 seconds';
-- END IF;
--
-- IF NEW.air_quality_update_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.air_quality_update_interval := '0 seconds';
-- END IF;
--
-- IF NEW.power_update_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.power_update_interval := '0 seconds';
-- END IF;
--
-- IF NEW.range_test_last_packet_timestamp < NOW() - INTERVAL '1 hours' THEN
-- NEW.range_test_interval := '0 seconds';
-- NEW.range_test_first_packet_timestamp := 0;
-- NEW.range_test_packets_total := 0;
-- END IF;
--
-- IF NEW.pax_counter_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.pax_counter_interval := '0 seconds';
-- END IF;
--
-- IF NEW.neighbor_info_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.neighbor_info_interval := '0 seconds';
-- END IF;
--
-- IF NEW.map_broadcast_last_timestamp < NOW() - INTERVAL '24 hours' THEN
-- NEW.map_broadcast_interval := '0 seconds';
-- END IF;
--
-- NEW.last_updated := CURRENT_TIMESTAMP;
--
-- RETURN NEW;
-- END;
-- $$ LANGUAGE plpgsql;
--
-- -- Create the trigger
-- CREATE TRIGGER update_node_configurations_trigger
-- BEFORE UPDATE ON node_configurations
-- FOR EACH ROW
-- EXECUTE FUNCTION update_old_node_configurations();

2
exporter/__init__.py
View file

@ -1 +1 @@
from .processor_base import MessageProcessor
from exporter.processor.processor_base import MessageProcessor

0
exporter/metric/__init__.py Normal file
View file

212
exporter/metric/node_configuration_metrics.py Normal file
View file

@ -0,0 +1,212 @@
import os
from psycopg_pool import ConnectionPool
from exporter.db_handler import DBHandler
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class NodeConfigurationMetrics(metaclass=Singleton):
def __init__(self, connection_pool: ConnectionPool = None):
self.db = DBHandler(connection_pool)
self.report = os.getenv('REPORT_NODE_CONFIGURATIONS', True)
def process_environment_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (node_id,
environment_update_interval,
environment_update_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
environment_update_interval = NOW() - node_configurations.environment_update_last_timestamp,
environment_update_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_device_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (node_id,
device_update_interval,
device_update_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
device_update_interval = NOW() - node_configurations.device_update_last_timestamp,
device_update_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_power_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (node_id,
power_update_interval,
power_update_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
power_update_interval = NOW() - node_configurations.power_update_last_timestamp,
power_update_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def map_broadcast_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (node_id,
map_broadcast_interval,
map_broadcast_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
map_broadcast_interval = NOW() - node_configurations.map_broadcast_last_timestamp,
map_broadcast_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_air_quality_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (node_id,
air_quality_update_interval,
air_quality_update_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
air_quality_update_interval = NOW() - node_configurations.air_quality_update_last_timestamp,
air_quality_update_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_range_test_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (
node_id,
range_test_interval,
range_test_packets_total,
range_test_first_packet_timestamp,
range_test_last_packet_timestamp
) VALUES (%s, %s, NOW(), NOW(), NOW())
ON CONFLICT(node_id)
DO UPDATE SET
range_test_interval = NOW() - node_configurations.range_test_last_packet_timestamp,
range_test_packets_total = CASE
WHEN EXCLUDED.range_test_last_packet_timestamp - node_configurations.range_test_first_packet_timestamp >= INTERVAL '1 hour'
THEN 1
ELSE node_configurations.range_test_packets_total + 1
END,
range_test_first_packet_timestamp = CASE
WHEN EXCLUDED.range_test_last_packet_timestamp - node_configurations.range_test_first_packet_timestamp >= INTERVAL '1 hour'
THEN NOW()
ELSE node_configurations.range_test_first_packet_timestamp
END,
range_test_last_packet_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_pax_counter_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (
node_id,
pax_counter_interval,
pax_counter_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
pax_counter_interval = NOW() - node_configurations.pax_counter_last_timestamp,
pax_counter_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_neighbor_info_update(self, node_id: str):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (
node_id,
neighbor_info_interval,
neighbor_info_last_timestamp
) VALUES (%s, %s, NOW())
ON CONFLICT(node_id)
DO UPDATE SET
neighbor_info_interval = NOW() - node_configurations.neighbor_info_last_timestamp,
neighbor_info_last_timestamp = NOW()
""", (node_id, '0 seconds'))
conn.commit()
self.db.execute_db_operation(db_operation)
def process_mqtt_update(self, node_id: str, mqtt_encryption_enabled=None, mqtt_json_enabled=None,
mqtt_configured_root_topic=None):
if not self.report:
return
def db_operation(cur, conn):
cur.execute("""
INSERT INTO node_configurations (
node_id,
mqtt_encryption_enabled,
mqtt_json_enabled,
mqtt_configured_root_topic,
mqtt_info_last_timestamp
) VALUES (%s, COALESCE(%s, FALSE), COALESCE(%s, FALSE), COALESCE(%s, ''), NOW())
ON CONFLICT(node_id)
DO UPDATE SET
mqtt_encryption_enabled = COALESCE(EXCLUDED.mqtt_encryption_enabled, node_configurations.mqtt_encryption_enabled),
mqtt_json_enabled = COALESCE(EXCLUDED.mqtt_json_enabled, node_configurations.mqtt_json_enabled),
mqtt_configured_root_topic = COALESCE(EXCLUDED.mqtt_configured_root_topic, node_configurations.mqtt_configured_root_topic),
mqtt_info_last_timestamp = NOW()
""", (node_id, mqtt_encryption_enabled, mqtt_json_enabled, mqtt_configured_root_topic))
conn.commit()
self.db.execute_db_operation(db_operation)

4
exporter/registry.py → exporter/metric/node_metrics.py
View file

@ -4,12 +4,12 @@ from exporter.client_details import ClientDetails
from exporter.db_handler import DBHandler
class _Metrics:
class Metrics:
_instance = None
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(_Metrics, cls).__new__(cls)
cls._instance = super(Metrics, cls).__new__(cls)
return cls._instance
def __init__(self, registry: CollectorRegistry, db: DBHandler):

0
exporter/processor/__init__.py Normal file
View file

32
exporter/processor_base.py → exporter/processor/processor_base.py
View file

@ -1,9 +1,13 @@
import base64
import json
import os
import sys
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from meshtastic.protobuf.mqtt_pb2 import ServiceEnvelope
from exporter.metric.node_configuration_metrics import NodeConfigurationMetrics
try:
from meshtastic.mesh_pb2 import MeshPacket, Data, HardwareModel
@ -16,7 +20,7 @@ from prometheus_client import CollectorRegistry, Counter, Gauge
from psycopg_pool import ConnectionPool
from exporter.client_details import ClientDetails
from exporter.processors import ProcessorRegistry
from exporter.processor.processors import ProcessorRegistry
class MessageProcessor:
@ -141,6 +145,32 @@ class MessageProcessor:
registry=self.registry
)
@staticmethod
def process_json_mqtt(message):
topic = message.topic
json_packet = json.loads(message.payload)
if json_packet['sender'][0] == '!':
gateway_node_id = str(int(json_packet['sender'][1:], 16))
NodeConfigurationMetrics().process_mqtt_update(
node_id=gateway_node_id,
mqtt_encryption_enabled=json_packet.get('encrypted', False),
mqtt_configured_root_topic=topic
)
@staticmethod
def process_mqtt(topic: str, service_envelope: ServiceEnvelope, mesh_packet: MeshPacket):
is_encrypted = False
if getattr(mesh_packet, 'encrypted'):
is_encrypted = True
if getattr(service_envelope, 'gateway_id'):
if service_envelope.gateway_id[0] == '!':
gateway_node_id = str(int(service_envelope.gateway_id[1:], 16))
NodeConfigurationMetrics().process_mqtt_update(
node_id=gateway_node_id,
mqtt_encryption_enabled=is_encrypted,
mqtt_configured_root_topic=topic
)
def process(self, mesh_packet: MeshPacket):
try:
if getattr(mesh_packet, 'encrypted'):

13
exporter/processors.py → exporter/processor/processors.py
View file

@ -6,6 +6,7 @@ import psycopg
import unishox2
from exporter.db_handler import DBHandler
from exporter.metric.node_configuration_metrics import NodeConfigurationMetrics
try:
from meshtastic.admin_pb2 import AdminMessage
@ -32,13 +33,13 @@ from prometheus_client import CollectorRegistry
from psycopg_pool import ConnectionPool
from exporter.client_details import ClientDetails
from exporter.registry import _Metrics
from exporter.metric.node_metrics import Metrics
class Processor(ABC):
def __init__(self, registry: CollectorRegistry, db_pool: ConnectionPool):
self.db_pool = db_pool
self.metrics = _Metrics(registry, DBHandler(db_pool))
self.metrics = Metrics(registry, DBHandler(db_pool))
@abstractmethod
def process(self, payload: bytes, client_details: ClientDetails):
@ -257,6 +258,7 @@ class IpTunnelAppProcessor(Processor):
class PaxCounterAppProcessor(Processor):
def process(self, payload: bytes, client_details: ClientDetails):
logger.debug("Received PAXCOUNTER_APP packet")
NodeConfigurationMetrics().process_pax_counter_update(client_details.node_id)
paxcounter = Paxcount()
try:
paxcounter.ParseFromString(payload)
@ -288,6 +290,7 @@ class StoreForwardAppProcessor(Processor):
class RangeTestAppProcessor(Processor):
def process(self, payload: bytes, client_details: ClientDetails):
logger.debug("Received RANGE_TEST_APP packet")
NodeConfigurationMetrics().process_range_test_update(client_details.node_id)
pass # NOTE: This portnum traffic is not sent to the public MQTT starting at firmware version 2.2.9
@ -306,6 +309,7 @@ class TelemetryAppProcessor(Processor):
return
if telemetry.HasField('device_metrics'):
NodeConfigurationMetrics().process_device_update(client_details.node_id)
device_metrics: DeviceMetrics = telemetry.device_metrics
self.metrics.battery_level_gauge.labels(
**client_details.to_dict()
@ -328,6 +332,7 @@ class TelemetryAppProcessor(Processor):
).inc(getattr(device_metrics, 'uptime_seconds', 0))
if telemetry.HasField('environment_metrics'):
NodeConfigurationMetrics().process_environment_update(client_details.node_id)
environment_metrics: EnvironmentMetrics = telemetry.environment_metrics
self.metrics.temperature_gauge.labels(
**client_details.to_dict()
@ -382,6 +387,7 @@ class TelemetryAppProcessor(Processor):
).set(getattr(environment_metrics, 'weight', 0))
if telemetry.HasField('air_quality_metrics'):
NodeConfigurationMetrics().process_air_quality_update(client_details.node_id)
air_quality_metrics: AirQualityMetrics = telemetry.air_quality_metrics
self.metrics.pm10_standard_gauge.labels(
**client_details.to_dict()
@ -432,6 +438,7 @@ class TelemetryAppProcessor(Processor):
).set(getattr(air_quality_metrics, 'particles_100um', 0))
if telemetry.HasField('power_metrics'):
NodeConfigurationMetrics().process_power_update(client_details.node_id)
power_metrics: PowerMetrics = telemetry.power_metrics
self.metrics.ch1_voltage_gauge.labels(
**client_details.to_dict()
@ -493,6 +500,7 @@ class TraceRouteAppProcessor(Processor):
class NeighborInfoAppProcessor(Processor):
def process(self, payload: bytes, client_details: ClientDetails):
logger.debug("Received NEIGHBORINFO_APP packet")
NodeConfigurationMetrics().process_neighbor_info_update(client_details.node_id)
neighbor_info = NeighborInfo()
try:
neighbor_info.ParseFromString(payload)
@ -547,6 +555,7 @@ class AtakPluginProcessor(Processor):
class MapReportAppProcessor(Processor):
def process(self, payload: bytes, client_details: ClientDetails):
logger.debug("Received MAP_REPORT_APP packet")
NodeConfigurationMetrics().map_broadcast_update(client_details.node_id)
map_report = MapReport()
try:
map_report.ParseFromString(payload)

16
main.py
View file

@ -6,6 +6,7 @@ import paho.mqtt.client as mqtt
from dotenv import load_dotenv
from constants import callback_api_version_map, protocol_map
from exporter.metric.node_configuration_metrics import NodeConfigurationMetrics
try:
from meshtastic.mesh_pb2 import MeshPacket
@ -38,9 +39,10 @@ def handle_connect(client, userdata, flags, reason_code, properties):
def update_node_status(node_number, status):
with connection_pool.connection() as conn:
with conn.cursor() as cur:
cur.execute("INSERT INTO node_details (node_id, mqtt_status) VALUES (%s, %s)"
cur.execute("INSERT INTO node_details (node_id, mqtt_status, short_name, long_name) VALUES (%s, %s, %s, %s)"
"ON CONFLICT(node_id)"
"DO UPDATE SET mqtt_status = %s", (node_number, status, status))
"DO UPDATE SET mqtt_status = %s",
(node_number, status, status, 'Unknown (MQTT)', 'Unknown (MQTT)'))
conn.commit()
@ -48,6 +50,7 @@ def handle_message(client, userdata, message):
current_timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"Received message on topic '{message.topic}' at {current_timestamp}")
if '/json/' in message.topic:
processor.process_json_mqtt(message)
# Ignore JSON messages as there are also protobuf messages sent on other topic
# Source: https://github.com/meshtastic/firmware/blob/master/src/mqtt/MQTT.cpp#L448
return
@ -78,7 +81,7 @@ def handle_message(client, userdata, message):
cur.execute("INSERT INTO messages (id, received_at) VALUES (%s, NOW()) ON CONFLICT (id) DO NOTHING",
(str(packet.id),))
conn.commit()
processor.process_mqtt(message.topic, envelope, packet)
processor.process(packet)
except Exception as e:
logging.error(f"Failed to handle message: {e}")
@ -89,14 +92,15 @@ if __name__ == "__main__":
load_dotenv()
# We have to load_dotenv before we can import MessageProcessor to allow filtering of message types
from exporter.processor_base import MessageProcessor
from exporter.processor.processor_base import MessageProcessor
# Setup a connection pool
connection_pool = ConnectionPool(
os.getenv('DATABASE_URL'),
min_size=1,
max_size=10
max_size=100
)
# Configure node configuration metrics
node_conf_metrics = NodeConfigurationMetrics(connection_pool)
# Configure Prometheus exporter
registry = CollectorRegistry()