From 877320784b8aa3b30835cd51eb95518886f7d697 Mon Sep 17 00:00:00 2001 From: fpetkovski Date: Wed, 23 Mar 2022 14:44:42 +0100 Subject: [PATCH 1/2] Add alert in mixin for exceeded sample limit This commit adds an alert in the prometheus mixin which triggers when Prometheus has failed scrapes that have exceeded the configured sample_limit for that job. Signed-off-by: fpetkovski --- documentation/prometheus-mixin/alerts.libsonnet | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/documentation/prometheus-mixin/alerts.libsonnet b/documentation/prometheus-mixin/alerts.libsonnet index 61328dc0a..bc77df55c 100644 --- a/documentation/prometheus-mixin/alerts.libsonnet +++ b/documentation/prometheus-mixin/alerts.libsonnet @@ -289,6 +289,20 @@ description: 'Prometheus %(prometheusName)s has dropped {{ printf "%%.0f" $value }} targets because some targets exceeded the configured body_size_limit.' % $._config, }, }, + { + alert: 'PrometheusScrapeSampleLimitHit', + expr: ||| + increase(prometheus_target_scrapes_exceeded_sample_limit_total{%(prometheusSelector)s}[5m]) > 0 + ||| % $._config, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Prometheus has failed scrapes that have exceeded the configured sample limit.', + description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes because some targets exceeded the configured sample_limit.' % $._config, + }, + }, { alert: 'PrometheusTargetSyncFailure', expr: ||| From 501a8a78654a58e349ced24a97e235c80926826f Mon Sep 17 00:00:00 2001 From: fpetkovski Date: Wed, 30 Mar 2022 09:35:03 +0200 Subject: [PATCH 2/2] Address code review comments Signed-off-by: fpetkovski --- documentation/prometheus-mixin/alerts.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/prometheus-mixin/alerts.libsonnet b/documentation/prometheus-mixin/alerts.libsonnet index bc77df55c..481bf6039 100644 --- a/documentation/prometheus-mixin/alerts.libsonnet +++ b/documentation/prometheus-mixin/alerts.libsonnet @@ -286,7 +286,7 @@ }, annotations: { summary: 'Prometheus has dropped some targets that exceeded body size limit.', - description: 'Prometheus %(prometheusName)s has dropped {{ printf "%%.0f" $value }} targets because some targets exceeded the configured body_size_limit.' % $._config, + description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.' % $._config, }, }, { @@ -300,7 +300,7 @@ }, annotations: { summary: 'Prometheus has failed scrapes that have exceeded the configured sample limit.', - description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes because some targets exceeded the configured sample_limit.' % $._config, + description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.' % $._config, }, }, {