From 820f8d595eeec7172b02289ba868bc3867c5f1b4 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Tue, 10 Sep 2019 16:52:12 +0200 Subject: [PATCH] docs/node-mixin: alert on desynchronised clock Signed-off-by: paulfantom --- docs/node-mixin/alerts/alerts.libsonnet | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 6c5ca573..6c2ee3e3 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -197,6 +197,44 @@ severity: 'warning', }, }, + { + alert: 'NodeClockSkewDetected', + expr: ||| + ( + node_timex_offset_seconds > 0.05 + and + deriv(node_timex_offset_seconds[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds < 0.05 + and + deriv(node_timex_offset_seconds[5m]) <= 0 + ) + ||| % $._config, + 'for': '10m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Clock skew detected.', + message: 'Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.', + }, + }, + { + alert: 'NodeClockNotSynchronising', + expr: ||| + min_over_time(node_timex_sync_status[5m]) == 0 + ||| % $._config, + 'for': '10m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Clock not synchronising.', + message: 'Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.', + }, + }, ], }, ],