load 5m http_requests{job="api-server", instance="0", group="production"} 0+10x10 http_requests{job="api-server", instance="1", group="production"} 0+20x10 http_requests{job="api-server", instance="0", group="canary"} 0+30x10 http_requests{job="api-server", instance="1", group="canary"} 0+40x10 http_requests{job="app-server", instance="0", group="production"} 0+50x10 http_requests{job="app-server", instance="1", group="production"} 0+60x10 http_requests{job="app-server", instance="0", group="canary"} 0+70x10 http_requests{job="app-server", instance="1", group="canary"} 0+80x10 load 5m vector_matching_a{l="x"} 0+1x100 vector_matching_a{l="y"} 0+2x50 vector_matching_b{l="x"} 0+4x25 eval instant at 50m SUM(http_requests) BY (job) - COUNT(http_requests) BY (job) {job="api-server"} 996 {job="app-server"} 2596 eval instant at 50m 2 - SUM(http_requests) BY (job) {job="api-server"} -998 {job="app-server"} -2598 eval instant at 50m 1000 / SUM(http_requests) BY (job) {job="api-server"} 1 {job="app-server"} 0.38461538461538464 eval instant at 50m SUM(http_requests) BY (job) - 2 {job="api-server"} 998 {job="app-server"} 2598 eval instant at 50m SUM(http_requests) BY (job) % 3 {job="api-server"} 1 {job="app-server"} 2 eval instant at 50m SUM(http_requests) BY (job) / 0 {job="api-server"} +Inf {job="app-server"} +Inf eval instant at 50m SUM(http_requests) BY (job) + SUM(http_requests) BY (job) {job="api-server"} 2000 {job="app-server"} 5200 eval instant at 50m http_requests{job="api-server", group="canary"} http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="1", job="api-server"} 400 eval instant at 50m http_requests{job="api-server", group="canary"} + rate(http_requests{job="api-server"}[5m]) * 5 * 60 {group="canary", instance="0", job="api-server"} 330 {group="canary", instance="1", job="api-server"} 440 eval instant at 50m rate(http_requests[25m]) * 25 * 60 {group="canary", instance="0", job="api-server"} 150 {group="canary", instance="0", job="app-server"} 350 {group="canary", instance="1", job="api-server"} 200 {group="canary", instance="1", job="app-server"} 400 {group="production", instance="0", job="api-server"} 50 {group="production", instance="0", job="app-server"} 249.99999999999997 {group="production", instance="1", job="api-server"} 100 {group="production", instance="1", job="app-server"} 300 eval instant at 50m http_requests{group="canary"} and http_requests{instance="0"} http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="app-server"} 700 eval instant at 50m (http_requests{group="canary"} + 1) and http_requests{instance="0"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 eval instant at 50m (http_requests{group="canary"} + 1) and on(instance, job) http_requests{instance="0", group="production"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 eval instant at 50m (http_requests{group="canary"} + 1) and on(instance) http_requests{instance="0", group="production"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group) http_requests{instance="0", group="production"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group, job) http_requests{instance="0", group="production"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 eval instant at 50m http_requests{group="canary"} or http_requests{group="production"} http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="app-server"} 700 http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 http_requests{group="production", instance="0", job="api-server"} 100 http_requests{group="production", instance="0", job="app-server"} 500 http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="app-server"} 600 # On overlap the rhs samples must be dropped. eval instant at 50m (http_requests{group="canary"} + 1) or http_requests{instance="1"} {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 {group="canary", instance="1", job="api-server"} 401 {group="canary", instance="1", job="app-server"} 801 http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="app-server"} 600 # Matching only on instance excludes everything that has instance=0/1 but includes # entries without the instance label. eval instant at 50m (http_requests{group="canary"} + 1) or on(instance) (http_requests or cpu_count or vector_matching_a) {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 {group="canary", instance="1", job="api-server"} 401 {group="canary", instance="1", job="app-server"} 801 vector_matching_a{l="x"} 10 vector_matching_a{l="y"} 20 eval instant at 50m (http_requests{group="canary"} + 1) or ignoring(l, group, job) (http_requests or cpu_count or vector_matching_a) {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 {group="canary", instance="1", job="api-server"} 401 {group="canary", instance="1", job="app-server"} 801 vector_matching_a{l="x"} 10 vector_matching_a{l="y"} 20 eval instant at 50m http_requests{group="canary"} unless http_requests{instance="0"} http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 eval instant at 50m http_requests{group="canary"} unless on(job) http_requests{instance="0"} eval instant at 50m http_requests{group="canary"} unless on(job, instance) http_requests{instance="0"} http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 eval instant at 50m http_requests{group="canary"} / on(instance,job) http_requests{group="production"} {instance="0", job="api-server"} 3 {instance="0", job="app-server"} 1.4 {instance="1", job="api-server"} 2 {instance="1", job="app-server"} 1.3333333333333333 eval instant at 50m http_requests{group="canary"} unless ignoring(group, instance) http_requests{instance="0"} eval instant at 50m http_requests{group="canary"} unless ignoring(group) http_requests{instance="0"} http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 eval instant at 50m http_requests{group="canary"} / ignoring(group) http_requests{group="production"} {instance="0", job="api-server"} 3 {instance="0", job="app-server"} 1.4 {instance="1", job="api-server"} 2 {instance="1", job="app-server"} 1.3333333333333333 # https://github.com/prometheus/prometheus/issues/1489 eval instant at 50m http_requests AND ON (dummy) vector(1) http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="app-server"} 700 http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 http_requests{group="production", instance="0", job="api-server"} 100 http_requests{group="production", instance="0", job="app-server"} 500 http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="app-server"} 600 eval instant at 50m http_requests AND IGNORING (group, instance, job) vector(1) http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="app-server"} 700 http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 http_requests{group="production", instance="0", job="api-server"} 100 http_requests{group="production", instance="0", job="app-server"} 500 http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="app-server"} 600 # Comparisons. eval instant at 50m SUM(http_requests) BY (job) > 1000 {job="app-server"} 2600 eval instant at 50m 1000 < SUM(http_requests) BY (job) {job="app-server"} 1000 eval instant at 50m SUM(http_requests) BY (job) <= 1000 {job="api-server"} 1000 eval instant at 50m SUM(http_requests) BY (job) != 1000 {job="app-server"} 2600 eval instant at 50m SUM(http_requests) BY (job) == 1000 {job="api-server"} 1000 eval instant at 50m SUM(http_requests) BY (job) == bool 1000 {job="api-server"} 1 {job="app-server"} 0 eval instant at 50m SUM(http_requests) BY (job) == bool SUM(http_requests) BY (job) {job="api-server"} 1 {job="app-server"} 1 eval instant at 50m SUM(http_requests) BY (job) != bool SUM(http_requests) BY (job) {job="api-server"} 0 {job="app-server"} 0 eval instant at 50m 0 == bool 1 0 eval instant at 50m 1 == bool 1 1 # group_left/group_right. clear load 5m node_var{instance="abc",job="node"} 2 node_role{instance="abc",job="node",role="prometheus"} 1 load 5m node_cpu{instance="abc",job="node",mode="idle"} 3 node_cpu{instance="abc",job="node",mode="user"} 1 node_cpu{instance="def",job="node",mode="idle"} 8 node_cpu{instance="def",job="node",mode="user"} 2 load 5m random{foo="bar"} 1 load 5m threshold{instance="abc",job="node",target="a@b.com"} 0 # Copy machine role to node variable. eval instant at 5m node_role * on (instance) group_right (role) node_var {instance="abc",job="node",role="prometheus"} 2 eval instant at 5m node_var * on (instance) group_left (role) node_role {instance="abc",job="node",role="prometheus"} 2 eval instant at 5m node_var * ignoring (role) group_left (role) node_role {instance="abc",job="node",role="prometheus"} 2 eval instant at 5m node_role * ignoring (role) group_right (role) node_var {instance="abc",job="node",role="prometheus"} 2 # Copy machine role to node variable with instrumentation labels. eval instant at 5m node_cpu * ignoring (role, mode) group_left (role) node_role {instance="abc",job="node",mode="idle",role="prometheus"} 3 {instance="abc",job="node",mode="user",role="prometheus"} 1 eval instant at 5m node_cpu * on (instance) group_left (role) node_role {instance="abc",job="node",mode="idle",role="prometheus"} 3 {instance="abc",job="node",mode="user",role="prometheus"} 1 # Ratio of total. eval instant at 5m node_cpu / on (instance) group_left sum by (instance,job)(node_cpu) {instance="abc",job="node",mode="idle"} .75 {instance="abc",job="node",mode="user"} .25 {instance="def",job="node",mode="idle"} .80 {instance="def",job="node",mode="user"} .20 eval instant at 5m sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu) {job="node",mode="idle"} 0.7857142857142857 {job="node",mode="user"} 0.21428571428571427 eval instant at 5m sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)) {} 1.0 eval instant at 5m node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu) {instance="abc",job="node",mode="idle"} .75 {instance="abc",job="node",mode="user"} .25 {instance="def",job="node",mode="idle"} .80 {instance="def",job="node",mode="user"} .20 eval instant at 5m node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu) {instance="abc",job="node",mode="idle"} .75 {instance="abc",job="node",mode="user"} .25 {instance="def",job="node",mode="idle"} .80 {instance="def",job="node",mode="user"} .20 eval instant at 5m sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu) {job="node",mode="idle"} 0.7857142857142857 {job="node",mode="user"} 0.21428571428571427 eval instant at 5m sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)) {} 1.0 # Copy over label from metric with no matching labels, without having to list cross-job target labels ('job' here). eval instant at 5m node_cpu + on(dummy) group_left(foo) random*0 {instance="abc",job="node",mode="idle",foo="bar"} 3 {instance="abc",job="node",mode="user",foo="bar"} 1 {instance="def",job="node",mode="idle",foo="bar"} 8 {instance="def",job="node",mode="user",foo="bar"} 2 # Use threshold from metric, and copy over target. eval instant at 5m node_cpu > on(job, instance) group_left(target) threshold node_cpu{instance="abc",job="node",mode="idle",target="a@b.com"} 3 node_cpu{instance="abc",job="node",mode="user",target="a@b.com"} 1 # Use threshold from metric, and a default (1) if it's not present. eval instant at 5m node_cpu > on(job, instance) group_left(target) (threshold or on (job, instance) (sum by (job, instance)(node_cpu) * 0 + 1)) node_cpu{instance="abc",job="node",mode="idle",target="a@b.com"} 3 node_cpu{instance="abc",job="node",mode="user",target="a@b.com"} 1 node_cpu{instance="def",job="node",mode="idle"} 8 node_cpu{instance="def",job="node",mode="user"} 2