node_exporter/docs/node-mixin/rules/rules.libsonnet
Julian Wiedmann 3e6f4ce627
mixin: exclude iowait and steal from CPU Utilisation (#2194)
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.

Per the iostat man page:

%idle
    Show the percentage of time that the CPU or CPUs were idle and the
    system did not have an outstanding disk I/O request.

%iowait
     Show the percentage of time that the CPU or CPUs were idle during
     which the system had an outstanding disk I/O request.

%steal
     Show the percentage of time spent in involuntary wait by the
     virtual CPU or CPUs while the hypervisor was servicing another
     virtual processor.

Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
2021-11-04 11:03:27 +01:00

120 lines
4.7 KiB
Plaintext

{
prometheusRules+:: {
groups+: [
{
name: 'node-exporter.rules',
rules: [
{
// This rule gives the number of CPUs per node.
record: 'instance:node_num_cpu:sum',
expr: |||
count without (cpu, mode) (
node_cpu_seconds_total{%(nodeExporterSelector)s,mode="idle"}
)
||| % $._config,
},
{
// CPU utilisation is % CPU without {idle,iowait,steal}.
record: 'instance:node_cpu_utilisation:rate%(rateInterval)s' % $._config,
expr: |||
1 - avg without (cpu) (
sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle|iowait|steal"}[%(rateInterval)s]))
)
||| % $._config,
},
{
// This is CPU saturation: 1min avg run queue length / number of CPUs.
// Can go over 1.
// TODO: There are situation where a run queue >1/core is just normal and fine.
// We need to clarify how to read this metric and if its usage is helpful at all.
record: 'instance:node_load1_per_cpu:ratio',
expr: |||
(
node_load1{%(nodeExporterSelector)s}
/
instance:node_num_cpu:sum{%(nodeExporterSelector)s}
)
||| % $._config,
},
{
// Memory utilisation (ratio of used memory per instance).
record: 'instance:node_memory_utilisation:ratio',
expr: |||
1 - (
(
node_memory_MemAvailable_bytes{%(nodeExporterSelector)s}
or
(
node_memory_Buffers_bytes{%(nodeExporterSelector)s}
+
node_memory_Cached_bytes{%(nodeExporterSelector)s}
+
node_memory_MemFree_bytes{%(nodeExporterSelector)s}
+
node_memory_Slab_bytes{%(nodeExporterSelector)s}
)
)
/
node_memory_MemTotal_bytes{%(nodeExporterSelector)s}
)
||| % $._config,
},
{
record: 'instance:node_vmstat_pgmajfault:rate%(rateInterval)s' % $._config,
expr: |||
rate(node_vmstat_pgmajfault{%(nodeExporterSelector)s}[%(rateInterval)s])
||| % $._config,
},
{
// Disk utilisation (seconds spent, 1 second rate).
record: 'instance_device:node_disk_io_time_seconds:rate%(rateInterval)s' % $._config,
expr: |||
rate(node_disk_io_time_seconds_total{%(nodeExporterSelector)s, %(diskDeviceSelector)s}[%(rateInterval)s])
||| % $._config,
},
{
// Disk saturation (weighted seconds spent, 1 second rate).
record: 'instance_device:node_disk_io_time_weighted_seconds:rate%(rateInterval)s' % $._config,
expr: |||
rate(node_disk_io_time_weighted_seconds_total{%(nodeExporterSelector)s, %(diskDeviceSelector)s}[%(rateInterval)s])
||| % $._config,
},
{
record: 'instance:node_network_receive_bytes_excluding_lo:rate%(rateInterval)s' % $._config,
expr: |||
sum without (device) (
rate(node_network_receive_bytes_total{%(nodeExporterSelector)s, device!="lo"}[%(rateInterval)s])
)
||| % $._config,
},
{
record: 'instance:node_network_transmit_bytes_excluding_lo:rate%(rateInterval)s' % $._config,
expr: |||
sum without (device) (
rate(node_network_transmit_bytes_total{%(nodeExporterSelector)s, device!="lo"}[%(rateInterval)s])
)
||| % $._config,
},
// TODO: Find out if those drops ever happen on modern switched networks.
{
record: 'instance:node_network_receive_drop_excluding_lo:rate%(rateInterval)s' % $._config,
expr: |||
sum without (device) (
rate(node_network_receive_drop_total{%(nodeExporterSelector)s, device!="lo"}[%(rateInterval)s])
)
||| % $._config,
},
{
record: 'instance:node_network_transmit_drop_excluding_lo:rate%(rateInterval)s' % $._config,
expr: |||
sum without (device) (
rate(node_network_transmit_drop_total{%(nodeExporterSelector)s, device!="lo"}[%(rateInterval)s])
)
||| % $._config,
},
],
},
],
},
}