From 55ffe57cbc00d278cf788ad98512c281ea891cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Blot?= Date: Fri, 30 Apr 2021 12:01:51 +0200 Subject: [PATCH] feat(rules): add NodeFileDescriptorLimit kernel exhaustion alert Add a new alert when fs.file-nr is close to fs.file-max Signed-off-by: Loic Blot --- docs/node-mixin/alerts/alerts.libsonnet | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index a19c8dc9..f39ef52c 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -277,6 +277,38 @@ description: "At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.", }, }, + { + alert: 'NodeFileDescriptorLimit', + expr: ||| + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + ||| % $._config, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Kernel is predicted to exhaust file descriptors limit soon.', + description: 'File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.', + }, + }, + { + alert: 'NodeFileDescriptorLimit', + expr: ||| + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + ||| % $._config, + 'for': '15m', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'Kernel is predicted to exhaust file descriptors limit soon.', + description: 'File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.', + }, + }, ], }, ],