From e5f94145b87d418ca4accca3f927e2aec5e1c0ef Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Tue, 23 May 2017 18:03:57 +0100 Subject: [PATCH] Drop series for federation if latest sample is stale. --- web/federate.go | 8 ++++++++ web/federate_test.go | 26 +++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/web/federate.go b/web/federate.go index bc79ffc91..26ea75c0d 100644 --- a/web/federate.go +++ b/web/federate.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" + "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" ) @@ -99,6 +100,13 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { continue } } + // The exposition formats do not support stale markers, so drop them. This + // is good enough for staleness handling of federated data, as the + // interval-based limits on staleness will do the right thing for supported + // use cases (which is to say federating aggregated time series). + if value.IsStaleNaN(v) { + continue + } vec = append(vec, promql.Sample{ Metric: s.Labels(), diff --git a/web/federate_test.go b/web/federate_test.go index 028b80709..765c8ec54 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -75,6 +75,18 @@ test_metric2{foo="boo",instance="i"} 1 6000000 code: 200, body: `# TYPE test_metric_without_labels untyped test_metric_without_labels{instance=""} 1001 6000000 +`, + }, + "test_stale_metric": { + params: "match[]=test_metric_stale", + code: 200, + body: ``, + }, + "test_old_metric": { + params: "match[]=test_metric_old", + code: 200, + body: `# TYPE test_metric_old untyped +test_metric_old{instance=""} 981 5880000 `, }, "{foo='boo'}": { @@ -104,6 +116,8 @@ test_metric1{foo="bar",instance="i"} 10000 6000000 test_metric1{foo="boo",instance="i"} 1 6000000 # TYPE test_metric2 untyped test_metric2{foo="boo",instance="i"} 1 6000000 +# TYPE test_metric_old untyped +test_metric_old{instance=""} 981 5880000 # TYPE test_metric_without_labels untyped test_metric_without_labels{instance=""} 1001 6000000 `, @@ -111,7 +125,9 @@ test_metric_without_labels{instance=""} 1001 6000000 "empty label value matches everything that doesn't have that label": { params: "match[]={foo='',__name__=~'.%2b'}", code: 200, - body: `# TYPE test_metric_without_labels untyped + body: `# TYPE test_metric_old untyped +test_metric_old{instance=""} 981 5880000 +# TYPE test_metric_without_labels untyped test_metric_without_labels{instance=""} 1001 6000000 `, }, @@ -123,6 +139,8 @@ test_metric1{foo="bar",instance="i"} 10000 6000000 test_metric1{foo="boo",instance="i"} 1 6000000 # TYPE test_metric2 untyped test_metric2{foo="boo",instance="i"} 1 6000000 +# TYPE test_metric_old untyped +test_metric_old{instance=""} 981 5880000 # TYPE test_metric_without_labels untyped test_metric_without_labels{instance=""} 1001 6000000 `, @@ -136,6 +154,8 @@ test_metric1{foo="bar",instance="i",zone="ie"} 10000 6000000 test_metric1{foo="boo",instance="i",zone="ie"} 1 6000000 # TYPE test_metric2 untyped test_metric2{foo="boo",instance="i",zone="ie"} 1 6000000 +# TYPE test_metric_old untyped +test_metric_old{foo="baz",instance="",zone="ie"} 981 5880000 # TYPE test_metric_without_labels untyped test_metric_without_labels{foo="baz",instance="",zone="ie"} 1001 6000000 `, @@ -151,6 +171,8 @@ test_metric1{foo="bar",instance="i"} 10000 6000000 test_metric1{foo="boo",instance="i"} 1 6000000 # TYPE test_metric2 untyped test_metric2{foo="boo",instance="i"} 1 6000000 +# TYPE test_metric_old untyped +test_metric_old{instance="baz"} 981 5880000 # TYPE test_metric_without_labels untyped test_metric_without_labels{instance="baz"} 1001 6000000 `, @@ -164,6 +186,8 @@ func TestFederation(t *testing.T) { test_metric1{foo="boo",instance="i"} 1+0x100 test_metric2{foo="boo",instance="i"} 1+0x100 test_metric_without_labels 1+10x100 + test_metric_stale 1+10x99 stale + test_metric_old 1+10x98 `) if err != nil { t.Fatal(err)