Merge pull request #4284 from prometheus/231

Release 2.3.1

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 2.3.1 / 2018-06-19
+
+* [BUGFIX] Avoid infinite loop on duplicate NaN values. #4275
+* [BUGFIX] Fix nil pointer deference when using various API endpoints #4282
+* [BUGFIX] config: set target group source index during unmarshalling #4245
+* [BUGFIX] discovery/file: fix logging #4178
+* [BUGFIX] kubernetes_sd: fix namespace filtering #4285
+* [BUGFIX] web: restore old path prefix behavior #4273
+* [BUGFIX] web: remove security headers added in 2.3.0 #4259
+
 ## 2.3.0 / 2018-06-05
 
 * [CHANGE] `marathon_sd`: use `auth_token` and `auth_token_file` for token-based authentication instead of `bearer_token` and `bearer_token_file` respectively.

VERSION

@@ -1 +1 @@
-2.3.0
+2.3.1

config/config.go

@@ -370,6 +370,13 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			}
 		}
 	}
+
+	// Add index to the static config target groups for unique identification
+	// within scrape pool.
+	for i, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
+		tg.Source = fmt.Sprintf("%d", i)
+	}
+
 	return nil
 }
 
@@ -432,6 +439,13 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
 			}
 		}
 	}
+
+	// Add index to the static config target groups for unique identification
+	// within scrape pool.
+	for i, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
+		tg.Source = fmt.Sprintf("%d", i)
+	}
+
 	return nil
 }
 

config/config_test.go

@@ -128,6 +128,7 @@ var expectedConf = &Config{
 							"my":   "label",
 							"your": "label",
 						},
+						Source: "0",
 					},
 				},
 
@@ -484,6 +485,7 @@ var expectedConf = &Config{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
 						},
+						Source: "0",
 					},
 				},
 			},
@@ -503,6 +505,7 @@ var expectedConf = &Config{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
 						},
+						Source: "0",
 					},
 				},
 			},
@@ -548,6 +551,7 @@ var expectedConf = &Config{
 								{model.AddressLabel: "1.2.3.5:9093"},
 								{model.AddressLabel: "1.2.3.6:9093"},
 							},
+							Source: "0",
 						},
 					},
 				},

discovery/file/file.go

@@ -279,7 +279,7 @@ func (d *Discovery) deleteTimestamp(filename string) {
 
 // stop shuts down the file watcher.
 func (d *Discovery) stop() {
-	level.Debug(d.logger).Log("msg", "Stopping file discovery...", "paths", d.paths)
+	level.Debug(d.logger).Log("msg", "Stopping file discovery...", "paths", fmt.Sprintf("%v", d.paths))
 
 	done := make(chan struct{})
 	defer close(done)
@@ -299,10 +299,10 @@ func (d *Discovery) stop() {
 		}
 	}()
 	if err := d.watcher.Close(); err != nil {
-		level.Error(d.logger).Log("msg", "Error closing file watcher", "paths", d.paths, "err", err)
+		level.Error(d.logger).Log("msg", "Error closing file watcher", "paths", fmt.Sprintf("%v", d.paths), "err", err)
 	}
 
-	level.Debug(d.logger).Log("File discovery stopped", "paths", d.paths)
+	level.Debug(d.logger).Log("msg", "File discovery stopped")
 }
 
 // refresh reads all files matching the discovery's patterns and sends the respective

discovery/manager.go

@@ -285,7 +285,7 @@ func (m *Manager) providersFromConfig(cfg sd_config.ServiceDiscoveryConfig) map[
 		app("triton", i, t)
 	}
 	if len(cfg.StaticConfigs) > 0 {
-		app("static", 0, NewStaticProvider(cfg.StaticConfigs))
+		app("static", 0, &StaticProvider{cfg.StaticConfigs})
 	}
 
 	return providers
@@ -296,15 +296,6 @@ type StaticProvider struct {
 	TargetGroups []*targetgroup.Group
 }
 
-// NewStaticProvider returns a StaticProvider configured with the given
-// target groups.
-func NewStaticProvider(groups []*targetgroup.Group) *StaticProvider {
-	for i, tg := range groups {
-		tg.Source = fmt.Sprintf("%d", i)
-	}
-	return &StaticProvider{groups}
-}
-
 // Run implements the Worker interface.
 func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	// We still have to consider that the consumer exits right away in which case

discovery/manager_test.go

@@ -774,6 +774,45 @@ scrape_configs:
 	verifyPresence(discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", false)
 }
 
+func TestApplyConfigDoesNotModifyStaticProviderTargets(t *testing.T) {
+	cfgText := `
+scrape_configs:
+ - job_name: 'prometheus'
+   static_configs:
+   - targets: ["foo:9090"]
+   - targets: ["bar:9090"]
+   - targets: ["baz:9090"]
+`
+	originalConfig := &config.Config{}
+	if err := yaml.UnmarshalStrict([]byte(cfgText), originalConfig); err != nil {
+		t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
+	}
+	origScrpCfg := originalConfig.ScrapeConfigs[0]
+
+	processedConfig := &config.Config{}
+	if err := yaml.UnmarshalStrict([]byte(cfgText), processedConfig); err != nil {
+		t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	discoveryManager := NewManager(ctx, nil)
+	go discoveryManager.Run()
+
+	c := make(map[string]sd_config.ServiceDiscoveryConfig)
+	for _, v := range processedConfig.ScrapeConfigs {
+		c[v.JobName] = v.ServiceDiscoveryConfig
+	}
+	discoveryManager.ApplyConfig(c)
+	<-discoveryManager.SyncCh()
+
+	for _, sdcfg := range c {
+		if !reflect.DeepEqual(origScrpCfg.ServiceDiscoveryConfig.StaticConfigs, sdcfg.StaticConfigs) {
+			t.Fatalf("discovery manager modified static config \n  expected: %v\n  got: %v\n",
+				origScrpCfg.ServiceDiscoveryConfig.StaticConfigs, sdcfg.StaticConfigs)
+		}
+	}
+}
+
 type update struct {
 	targetGroups []targetgroup.Group
 	interval     time.Duration

storage/remote/codec.go

@@ -92,9 +92,12 @@ func ToQuery(from, to int64, matchers []*labels.Matcher, p *storage.SelectParams
 		return nil, err
 	}
 
-	rp := &prompb.ReadHints{
-		StepMs: p.Step,
-		Func:   p.Func,
+	var rp *prompb.ReadHints
+	if p != nil {
+		rp = &prompb.ReadHints{
+			StepMs: p.Step,
+			Func:   p.Func,
+		}
 	}
 
 	return &prompb.Query{

web/api/v1/api_test.go

@@ -31,7 +31,9 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promlog"
 	"github.com/prometheus/common/route"
 
 	"github.com/prometheus/prometheus/config"
@@ -128,30 +130,125 @@ func TestEndpoints(t *testing.T) {
 
 	now := time.Now()
 
-	var tr testTargetRetriever
+	t.Run("local", func(t *testing.T) {
+		api := &API{
+			Queryable:             suite.Storage(),
+			QueryEngine:           suite.QueryEngine(),
+			targetRetriever:       testTargetRetriever{},
+			alertmanagerRetriever: testAlertmanagerRetriever{},
+			now:      func() time.Time { return now },
+			config:   func() config.Config { return samplePrometheusCfg },
+			flagsMap: sampleFlagMap,
+			ready:    func(f http.HandlerFunc) http.HandlerFunc { return f },
+		}
 
-	var ar testAlertmanagerRetriever
+		testEndpoints(t, api, true)
+	})
 
-	api := &API{
-		Queryable:             suite.Storage(),
-		QueryEngine:           suite.QueryEngine(),
-		targetRetriever:       tr,
-		alertmanagerRetriever: ar,
-		now:      func() time.Time { return now },
-		config:   func() config.Config { return samplePrometheusCfg },
-		flagsMap: sampleFlagMap,
-		ready:    func(f http.HandlerFunc) http.HandlerFunc { return f },
-	}
+	// Run all the API tests against a API that is wired to forward queries via
+	// the remote read client to a test server, which in turn sends them to the
+	// data from the test suite.
+	t.Run("remote", func(t *testing.T) {
+		server := setupRemote(suite.Storage())
+		defer server.Close()
+
+		u, err := url.Parse(server.URL)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		al := promlog.AllowedLevel{}
+		al.Set("debug")
+		remote := remote.NewStorage(promlog.New(al), func() (int64, error) {
+			return 0, nil
+		}, 1*time.Second)
+
+		err = remote.ApplyConfig(&config.Config{
+			RemoteReadConfigs: []*config.RemoteReadConfig{
+				{
+					URL:           &config_util.URL{URL: u},
+					RemoteTimeout: model.Duration(1 * time.Second),
+					ReadRecent:    true,
+				},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		api := &API{
+			Queryable:             remote,
+			QueryEngine:           suite.QueryEngine(),
+			targetRetriever:       testTargetRetriever{},
+			alertmanagerRetriever: testAlertmanagerRetriever{},
+			now:      func() time.Time { return now },
+			config:   func() config.Config { return samplePrometheusCfg },
+			flagsMap: sampleFlagMap,
+			ready:    func(f http.HandlerFunc) http.HandlerFunc { return f },
+		}
+
+		testEndpoints(t, api, false)
+	})
+}
+
+func setupRemote(s storage.Storage) *httptest.Server {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		req, err := remote.DecodeReadRequest(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		resp := prompb.ReadResponse{
+			Results: make([]*prompb.QueryResult, len(req.Queries)),
+		}
+		for i, query := range req.Queries {
+			from, through, matchers, err := remote.FromQuery(query)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+
+			querier, err := s.Querier(r.Context(), from, through)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			defer querier.Close()
+
+			set, err := querier.Select(nil, matchers...)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			resp.Results[i], err = remote.ToQueryResult(set)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		if err := remote.EncodeReadResponse(&resp, w); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	})
+
+	return httptest.NewServer(handler)
+}
+
+func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
 
 	start := time.Unix(0, 0)
 
-	var tests = []struct {
+	type test struct {
 		endpoint apiFunc
 		params   map[string]string
 		query    url.Values
 		response interface{}
 		errType  errorType
-	}{
+	}
+
+	var tests = []test{
 		{
 			endpoint: api.query,
 			query: url.Values{
@@ -203,7 +300,7 @@ func TestEndpoints(t *testing.T) {
 				ResultType: promql.ValueTypeScalar,
 				Result: promql.Scalar{
 					V: 0.333,
-					T: timestamp.FromTime(now),
+					T: timestamp.FromTime(api.now()),
 				},
 			},
 		},
@@ -309,34 +406,6 @@ func TestEndpoints(t *testing.T) {
 			},
 			errType: errorBadData,
 		},
-		{
-			endpoint: api.labelValues,
-			params: map[string]string{
-				"name": "__name__",
-			},
-			response: []string{
-				"test_metric1",
-				"test_metric2",
-			},
-		},
-		{
-			endpoint: api.labelValues,
-			params: map[string]string{
-				"name": "foo",
-			},
-			response: []string{
-				"bar",
-				"boo",
-			},
-		},
-		// Bad name parameter.
-		{
-			endpoint: api.labelValues,
-			params: map[string]string{
-				"name": "not!!!allowed",
-			},
-			errType: errorBadData,
-		},
 		{
 			endpoint: api.series,
 			query: url.Values{
@@ -500,6 +569,39 @@ func TestEndpoints(t *testing.T) {
 		},
 	}
 
+	if testLabelAPI {
+		tests = append(tests, []test{
+			{
+				endpoint: api.labelValues,
+				params: map[string]string{
+					"name": "__name__",
+				},
+				response: []string{
+					"test_metric1",
+					"test_metric2",
+				},
+			},
+			{
+				endpoint: api.labelValues,
+				params: map[string]string{
+					"name": "foo",
+				},
+				response: []string{
+					"bar",
+					"boo",
+				},
+			},
+			// Bad name parameter.
+			{
+				endpoint: api.labelValues,
+				params: map[string]string{
+					"name": "not!!!allowed",
+				},
+				errType: errorBadData,
+			},
+		}...)
+	}
+
 	methods := func(f apiFunc) []string {
 		fp := reflect.ValueOf(f).Pointer()
 		if fp == reflect.ValueOf(api.query).Pointer() || fp == reflect.ValueOf(api.queryRange).Pointer() {
@@ -517,14 +619,14 @@ func TestEndpoints(t *testing.T) {
 		return http.NewRequest(m, fmt.Sprintf("http://example.com?%s", q.Encode()), nil)
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
 		for _, method := range methods(test.endpoint) {
 			// Build a context with the correct request params.
 			ctx := context.Background()
 			for p, v := range test.params {
 				ctx = route.WithParam(ctx, p, v)
 			}
-			t.Logf("run %s\t%q", method, test.query.Encode())
+			t.Logf("run %d\t%s\t%q", i, method, test.query.Encode())
 
 			req, err := request(method, test.query)
 			if err != nil {