http_sd (#8839)

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

config/config_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/eureka"
 	"github.com/prometheus/prometheus/discovery/file"
 	"github.com/prometheus/prometheus/discovery/hetzner"
+	"github.com/prometheus/prometheus/discovery/http"
 	"github.com/prometheus/prometheus/discovery/kubernetes"
 	"github.com/prometheus/prometheus/discovery/linode"
 	"github.com/prometheus/prometheus/discovery/marathon"
@@ -626,6 +627,25 @@ var expectedConf = &Config{
 				},
 			},
 		},
+		{
+			JobName: "httpsd",
+
+			HonorTimestamps: true,
+			ScrapeInterval:  model.Duration(15 * time.Second),
+			ScrapeTimeout:   DefaultGlobalConfig.ScrapeTimeout,
+
+			MetricsPath:      DefaultScrapeConfig.MetricsPath,
+			Scheme:           DefaultScrapeConfig.Scheme,
+			HTTPClientConfig: config.DefaultHTTPClientConfig,
+
+			ServiceDiscoveryConfigs: discovery.Configs{
+				&http.SDConfig{
+					HTTPClientConfig: config.DefaultHTTPClientConfig,
+					URL:              "http://example.com/prometheus",
+					RefreshInterval:  model.Duration(60 * time.Second),
+				},
+			},
+		},
 		{
 			JobName: "service-triton",
 
@@ -1233,6 +1253,18 @@ var expectedErrors = []struct {
 		filename: "scrape_body_size_limit.bad.yml",
 		errMsg:   "units: unknown unit  in 100",
 	},
+	{
+		filename: "http_url_no_scheme.bad.yml",
+		errMsg:   "URL scheme must be 'http' or 'https'",
+	},
+	{
+		filename: "http_url_no_host.bad.yml",
+		errMsg:   "host is missing in URL",
+	},
+	{
+		filename: "http_url_bad_scheme.bad.yml",
+		errMsg:   "URL scheme must be 'http' or 'https'",
+	},
 }
 
 func TestBadConfigs(t *testing.T) {

config/testdata/conf.good.yml

@@ -264,6 +264,10 @@ scrape_configs:
     - targets:
       - localhost:9090
 
+- job_name: httpsd
+  http_sd_configs:
+    - url: 'http://example.com/prometheus'
+
 - job_name: service-triton
   triton_sd_configs:
   - account: 'testAccount'

config/testdata/http_url_bad_scheme.bad.yml

@@ -0,0 +1,3 @@
+scrape_configs:
+- http_sd_configs:
+  - url: ftp://example.com

config/testdata/http_url_no_host.bad.yml

@@ -0,0 +1,3 @@
+scrape_configs:
+- http_sd_configs:
+  - url: http://

config/testdata/http_url_no_scheme.bad.yml

@@ -0,0 +1,3 @@
+scrape_configs:
+- http_sd_configs:
+    - url: invalid

discovery/http/http.go

@@ -0,0 +1,188 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/version"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/refresh"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+var (
+	// DefaultSDConfig is the default HTTP SD configuration.
+	DefaultSDConfig = SDConfig{
+		RefreshInterval:  model.Duration(60 * time.Second),
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+	}
+	userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
+)
+
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
+// SDConfig is the configuration for HTTP based discovery.
+type SDConfig struct {
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
+	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
+	URL              string                  `yaml:"url"`
+}
+
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "http" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultSDConfig
+	type plain SDConfig
+	err := unmarshal((*plain)(c))
+	if err != nil {
+		return err
+	}
+	if c.URL == "" {
+		return fmt.Errorf("URL is missing")
+	}
+	parsedURL, err := url.Parse(c.URL)
+	if err != nil {
+		return err
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return fmt.Errorf("URL scheme must be 'http' or 'https'")
+	}
+	if parsedURL.Host == "" {
+		return fmt.Errorf("host is missing in URL")
+	}
+	return nil
+}
+
+const httpSDURLLabel = model.MetaLabelPrefix + "url"
+
+// Discovery provides service discovery functionality based
+// on HTTP endpoints that return target groups in JSON format.
+type Discovery struct {
+	*refresh.Discovery
+	url             string
+	client          *http.Client
+	refreshInterval time.Duration
+}
+
+// NewDiscovery returns a new HTTP discovery for the given config.
+func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
+
+	client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", config.WithHTTP2Disabled())
+	if err != nil {
+		return nil, err
+	}
+	client.Timeout = time.Duration(conf.RefreshInterval)
+
+	d := &Discovery{
+		url:             conf.URL,
+		client:          client,
+		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
+	}
+
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"http",
+		time.Duration(conf.RefreshInterval),
+		d.refresh,
+	)
+	return d, nil
+}
+
+func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	req, err := http.NewRequest("GET", d.url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("X-Prometheus-Refresh-Interval-Seconds", fmt.Sprintf("%f", d.refreshInterval.Seconds()))
+
+	resp, err := d.client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("server returned HTTP status %s", resp.Status)
+	}
+
+	if resp.Header.Get("Content-Type") != "application/json" {
+		return nil, errors.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var targetGroups []*targetgroup.Group
+
+	if err := json.Unmarshal(b, &targetGroups); err != nil {
+		return nil, err
+	}
+
+	for i, tg := range targetGroups {
+		if tg == nil {
+			err = errors.New("nil target group item found")
+			return nil, err
+		}
+
+		tg.Source = urlSource(d.url, i)
+		if tg.Labels == nil {
+			tg.Labels = model.LabelSet{}
+		}
+		tg.Labels[httpSDURLLabel] = model.LabelValue(d.url)
+	}
+
+	return targetGroups, nil
+}
+
+// urlSource returns a source ID for the i-th target group per URL.
+func urlSource(url string, i int) string {
+	return fmt.Sprintf("%s:%d", url, i)
+}

discovery/install/install.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/file"         // register file
 	_ "github.com/prometheus/prometheus/discovery/gce"          // register gce
 	_ "github.com/prometheus/prometheus/discovery/hetzner"      // register hetzner
+	_ "github.com/prometheus/prometheus/discovery/http"         // register http
 	_ "github.com/prometheus/prometheus/discovery/kubernetes"   // register kubernetes
 	_ "github.com/prometheus/prometheus/discovery/linode"       // register linode
 	_ "github.com/prometheus/prometheus/discovery/marathon"     // register marathon

docs/configuration/configuration.md

@@ -240,6 +240,10 @@ gce_sd_configs:
 hetzner_sd_configs:
   [ - <hetzner_sd_config> ... ]
 
+# List of HTTP service discovery configurations.
+http_sd_configs:
+  [ - <http_sd_config> ... ]
+
 # List of Kubernetes service discovery configurations.
 kubernetes_sd_configs:
   [ - <kubernetes_sd_config> ... ]
@@ -343,7 +347,7 @@ A `tls_config` allows configuring TLS connections.
 [ insecure_skip_verify: <boolean> ]
 ```
 
-### `oauth2`
+### `<oauth2>`
 
 OAuth 2.0 authentication using the client credentials grant type.
 Prometheus fetches an access token from the specified endpoint with
@@ -524,14 +528,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -621,14 +625,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -784,14 +788,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -1196,7 +1200,7 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration. required when role is
+# Optional `Authorization` header configuration, required when role is
 # hcloud. Role robot does not support bearer token authentication.
 authorization:
   # Sets the authentication type.
@@ -1204,7 +1208,7 @@ authorization:
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -1230,6 +1234,81 @@ tls_config:
 [ refresh_interval: <duration> | default = 60s ]
 ```
 
+### `<http_sd_config>`
+
+HTTP-based service discovery provides a more generic way to configure static targets
+and serves as an interface to plug in custom service discovery mechanisms.
+
+It fetches targets from an HTTP endpoint containing a list of zero or more
+`<static_config>`s. The target must reply with an HTTP 200 response.
+The HTTP header `Content-Type` must be `application/json`, and the body must be
+valid JSON.
+
+Example response body:
+
+```json
+[
+  {
+    "targets": [ "<host>", ... ],
+    "labels": {
+      "<labelname>": "<labelvalue>", ...
+    }
+  },
+  ...
+]
+```
+
+The endpoint is queried periodically at the specified
+refresh interval.
+
+Each target has a meta label `__meta_url` during the
+[relabeling phase](#relabel_config). Its value is set to the
+URL from which the target was extracted.
+
+```yaml
+# URL from wich the targets are fetched.
+url: <string>
+
+# Refresh interval to re-query the endpoint.
+[ refresh_interval: <duration> | default = 60s ]
+
+# Authentication information used to authenticate to the API server.
+# Note that `basic_auth`, `authorization` and `oauth2` options are
+# mutually exclusive.
+# `password` and `password_file` are mutually exclusive.
+
+# Optional HTTP basic authentication information.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# Optional `Authorization` header configuration.
+authorization:
+  # Sets the authentication type.
+  [ type: <string> | default: Bearer ]
+  # Sets the credentials. It is mutually exclusive with
+  # `credentials_file`.
+  [ credentials: <secret> ]
+  # Sets the credentials to the credentials read from the configured file.
+  # It is mutually exclusive with `credentials`.
+  [ credentials_file: <filename> ]
+
+# Optional OAuth 2.0 configuration.
+oauth2:
+  [ <oauth2> ]
+
+# Optional proxy URL.
+[ proxy_url: <string> ]
+
+# Configure whether HTTP requests follow HTTP 3xx redirects.
+[ follow_redirects: <bool> | default = true ]
+
+# TLS configuration.
+tls_config:
+  [ <tls_config> ]
+```
+
 ### `<kubernetes_sd_config>`
 
 Kubernetes SD configurations allow retrieving scrape targets from
@@ -1373,14 +1452,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -1601,7 +1680,7 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 # NOTE: The current version of DC/OS marathon (v1.11.0) does not support
 # standard `Authentication` header, use `auth_token` or `auth_token_file`
 # instead.
@@ -1611,7 +1690,7 @@ authorization:
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -1807,14 +1886,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -2090,14 +2169,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -2160,6 +2239,10 @@ gce_sd_configs:
 hetzner_sd_configs:
   [ - <hetzner_sd_config> ... ]
 
+# List of HTTP service discovery configurations.
+http_sd_configs:
+  [ - <http_sd_config> ... ]
+
 # List of Kubernetes service discovery configurations.
 kubernetes_sd_configs:
   [ - <kubernetes_sd_config> ... ]
@@ -2246,14 +2329,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
@@ -2364,14 +2447,14 @@ basic_auth:
   [ password: <secret> ]
   [ password_file: <string> ]
 
-# Optional the `Authorization` header configuration.
+# Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
   [ type: <string> | default: Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
-  # Sets the credentials with the credentials read from the configured file.
+  # Sets the credentials to the credentials read from the configured file.
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]