diff --git a/documentation/prometheus-mixin/alerts.libsonnet b/documentation/prometheus-mixin/alerts.libsonnet index 4c66c4a9ba..08b43b4ab3 100644 --- a/documentation/prometheus-mixin/alerts.libsonnet +++ b/documentation/prometheus-mixin/alerts.libsonnet @@ -156,6 +156,22 @@ message: 'Prometheus failed to send {{ printf "%.1f" $value }}% samples', }, }, + { + alert: 'PrometheusRemoteWriteBehind', + expr: ||| + prometheus_remote_storage_highest_timestamp_in{%(prometheusSelector)s} + - on(job, instance) group_right + prometheus_remote_storage_queue_highest_sent_timestamp{%(prometheusSelector)s} + > 60 + ||| % $._config, + 'for': '15m', + labels: { + severity: 'critical', + }, + annotations: { + message: 'Prometheus remote write is {{ printf "%.1f" $value }}s behind.', + }, + }, { alert: 'PrometheusRuleFailures', 'for': '15m',