From 9444698ae232193ab4b5e41f51b62648919a5db5 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 11 Jun 2021 18:04:45 +0200 Subject: [PATCH] http_sd (#8839) Signed-off-by: Julien Pivotto --- config/config_test.go | 32 ++++ config/testdata/conf.good.yml | 4 + config/testdata/http_url_bad_scheme.bad.yml | 3 + config/testdata/http_url_no_host.bad.yml | 3 + config/testdata/http_url_no_scheme.bad.yml | 3 + discovery/http/http.go | 188 ++++++++++++++++++++ discovery/install/install.go | 1 + docs/configuration/configuration.md | 125 ++++++++++--- 8 files changed, 338 insertions(+), 21 deletions(-) create mode 100644 config/testdata/http_url_bad_scheme.bad.yml create mode 100644 config/testdata/http_url_no_host.bad.yml create mode 100644 config/testdata/http_url_no_scheme.bad.yml create mode 100644 discovery/http/http.go diff --git a/config/config_test.go b/config/config_test.go index b9dbc750f..90522090c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,6 +39,7 @@ import ( "github.com/prometheus/prometheus/discovery/eureka" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/hetzner" + "github.com/prometheus/prometheus/discovery/http" "github.com/prometheus/prometheus/discovery/kubernetes" "github.com/prometheus/prometheus/discovery/linode" "github.com/prometheus/prometheus/discovery/marathon" @@ -626,6 +627,25 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "httpsd", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + + ServiceDiscoveryConfigs: discovery.Configs{ + &http.SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: "http://example.com/prometheus", + RefreshInterval: model.Duration(60 * time.Second), + }, + }, + }, { JobName: "service-triton", @@ -1233,6 +1253,18 @@ var expectedErrors = []struct { filename: "scrape_body_size_limit.bad.yml", errMsg: "units: unknown unit in 100", }, + { + filename: "http_url_no_scheme.bad.yml", + errMsg: "URL scheme must be 'http' or 'https'", + }, + { + filename: "http_url_no_host.bad.yml", + errMsg: "host is missing in URL", + }, + { + filename: "http_url_bad_scheme.bad.yml", + errMsg: "URL scheme must be 'http' or 'https'", + }, } func TestBadConfigs(t *testing.T) { diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 078510f93..61c3070ac 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -264,6 +264,10 @@ scrape_configs: - targets: - localhost:9090 +- job_name: httpsd + http_sd_configs: + - url: 'http://example.com/prometheus' + - job_name: service-triton triton_sd_configs: - account: 'testAccount' diff --git a/config/testdata/http_url_bad_scheme.bad.yml b/config/testdata/http_url_bad_scheme.bad.yml new file mode 100644 index 000000000..eca8024c0 --- /dev/null +++ b/config/testdata/http_url_bad_scheme.bad.yml @@ -0,0 +1,3 @@ +scrape_configs: +- http_sd_configs: + - url: ftp://example.com diff --git a/config/testdata/http_url_no_host.bad.yml b/config/testdata/http_url_no_host.bad.yml new file mode 100644 index 000000000..e1ee14d87 --- /dev/null +++ b/config/testdata/http_url_no_host.bad.yml @@ -0,0 +1,3 @@ +scrape_configs: +- http_sd_configs: + - url: http:// diff --git a/config/testdata/http_url_no_scheme.bad.yml b/config/testdata/http_url_no_scheme.bad.yml new file mode 100644 index 000000000..bb6fc8384 --- /dev/null +++ b/config/testdata/http_url_no_scheme.bad.yml @@ -0,0 +1,3 @@ +scrape_configs: +- http_sd_configs: + - url: invalid diff --git a/discovery/http/http.go b/discovery/http/http.go new file mode 100644 index 000000000..f02caa0f2 --- /dev/null +++ b/discovery/http/http.go @@ -0,0 +1,188 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "time" + + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +var ( + // DefaultSDConfig is the default HTTP SD configuration. + DefaultSDConfig = SDConfig{ + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, + } + userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +) + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// SDConfig is the configuration for HTTP based discovery. +type SDConfig struct { + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + URL string `yaml:"url"` +} + +// Name returns the name of the Config. +func (*SDConfig) Name() string { return "http" } + +// NewDiscoverer returns a Discoverer for the Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts.Logger) +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.URL == "" { + return fmt.Errorf("URL is missing") + } + parsedURL, err := url.Parse(c.URL) + if err != nil { + return err + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return fmt.Errorf("URL scheme must be 'http' or 'https'") + } + if parsedURL.Host == "" { + return fmt.Errorf("host is missing in URL") + } + return nil +} + +const httpSDURLLabel = model.MetaLabelPrefix + "url" + +// Discovery provides service discovery functionality based +// on HTTP endpoints that return target groups in JSON format. +type Discovery struct { + *refresh.Discovery + url string + client *http.Client + refreshInterval time.Duration +} + +// NewDiscovery returns a new HTTP discovery for the given config. +func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { + if logger == nil { + logger = log.NewNopLogger() + } + + client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", config.WithHTTP2Disabled()) + if err != nil { + return nil, err + } + client.Timeout = time.Duration(conf.RefreshInterval) + + d := &Discovery{ + url: conf.URL, + client: client, + refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers. + } + + d.Discovery = refresh.NewDiscovery( + logger, + "http", + time.Duration(conf.RefreshInterval), + d.refresh, + ) + return d, nil +} + +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + req, err := http.NewRequest("GET", d.url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Prometheus-Refresh-Interval-Seconds", fmt.Sprintf("%f", d.refreshInterval.Seconds())) + + resp, err := d.client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer func() { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return nil, errors.Errorf("server returned HTTP status %s", resp.Status) + } + + if resp.Header.Get("Content-Type") != "application/json" { + return nil, errors.Errorf("unsupported content type %q", resp.Header.Get("Content-Type")) + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var targetGroups []*targetgroup.Group + + if err := json.Unmarshal(b, &targetGroups); err != nil { + return nil, err + } + + for i, tg := range targetGroups { + if tg == nil { + err = errors.New("nil target group item found") + return nil, err + } + + tg.Source = urlSource(d.url, i) + if tg.Labels == nil { + tg.Labels = model.LabelSet{} + } + tg.Labels[httpSDURLLabel] = model.LabelValue(d.url) + } + + return targetGroups, nil +} + +// urlSource returns a source ID for the i-th target group per URL. +func urlSource(url string, i int) string { + return fmt.Sprintf("%s:%d", url, i) +} diff --git a/discovery/install/install.go b/discovery/install/install.go index ce20615e2..6258d6fc0 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -25,6 +25,7 @@ import ( _ "github.com/prometheus/prometheus/discovery/file" // register file _ "github.com/prometheus/prometheus/discovery/gce" // register gce _ "github.com/prometheus/prometheus/discovery/hetzner" // register hetzner + _ "github.com/prometheus/prometheus/discovery/http" // register http _ "github.com/prometheus/prometheus/discovery/kubernetes" // register kubernetes _ "github.com/prometheus/prometheus/discovery/linode" // register linode _ "github.com/prometheus/prometheus/discovery/marathon" // register marathon diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index a5a418fa7..0036132dd 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -240,6 +240,10 @@ gce_sd_configs: hetzner_sd_configs: [ - ... ] +# List of HTTP service discovery configurations. +http_sd_configs: + [ - ... ] + # List of Kubernetes service discovery configurations. kubernetes_sd_configs: [ - ... ] @@ -343,7 +347,7 @@ A `tls_config` allows configuring TLS connections. [ insecure_skip_verify: ] ``` -### `oauth2` +### `` OAuth 2.0 authentication using the client credentials grant type. Prometheus fetches an access token from the specified endpoint with @@ -524,14 +528,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -621,14 +625,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -784,14 +788,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -1196,7 +1200,7 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. required when role is +# Optional `Authorization` header configuration, required when role is # hcloud. Role robot does not support bearer token authentication. authorization: # Sets the authentication type. @@ -1204,7 +1208,7 @@ authorization: # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -1230,6 +1234,81 @@ tls_config: [ refresh_interval: | default = 60s ] ``` +### `` + +HTTP-based service discovery provides a more generic way to configure static targets +and serves as an interface to plug in custom service discovery mechanisms. + +It fetches targets from an HTTP endpoint containing a list of zero or more +``s. The target must reply with an HTTP 200 response. +The HTTP header `Content-Type` must be `application/json`, and the body must be +valid JSON. + +Example response body: + +```json +[ + { + "targets": [ "", ... ], + "labels": { + "": "", ... + } + }, + ... +] +``` + +The endpoint is queried periodically at the specified +refresh interval. + +Each target has a meta label `__meta_url` during the +[relabeling phase](#relabel_config). Its value is set to the +URL from which the target was extracted. + +```yaml +# URL from wich the targets are fetched. +url: + +# Refresh interval to re-query the endpoint. +[ refresh_interval: | default = 60s ] + +# Authentication information used to authenticate to the API server. +# Note that `basic_auth`, `authorization` and `oauth2` options are +# mutually exclusive. +# `password` and `password_file` are mutually exclusive. + +# Optional HTTP basic authentication information. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional `Authorization` header configuration. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials to the credentials read from the configured file. + # It is mutually exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration. +oauth2: + [ ] + +# Optional proxy URL. +[ proxy_url: ] + +# Configure whether HTTP requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] + +# TLS configuration. +tls_config: + [ ] +``` + ### `` Kubernetes SD configurations allow retrieving scrape targets from @@ -1373,14 +1452,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -1601,7 +1680,7 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. # NOTE: The current version of DC/OS marathon (v1.11.0) does not support # standard `Authentication` header, use `auth_token` or `auth_token_file` # instead. @@ -1611,7 +1690,7 @@ authorization: # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -1807,14 +1886,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -2090,14 +2169,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -2160,6 +2239,10 @@ gce_sd_configs: hetzner_sd_configs: [ - ... ] +# List of HTTP service discovery configurations. +http_sd_configs: + [ - ... ] + # List of Kubernetes service discovery configurations. kubernetes_sd_configs: [ - ... ] @@ -2246,14 +2329,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ] @@ -2364,14 +2447,14 @@ basic_auth: [ password: ] [ password_file: ] -# Optional the `Authorization` header configuration. +# Optional `Authorization` header configuration. authorization: # Sets the authentication type. [ type: | default: Bearer ] # Sets the credentials. It is mutually exclusive with # `credentials_file`. [ credentials: ] - # Sets the credentials with the credentials read from the configured file. + # Sets the credentials to the credentials read from the configured file. # It is mutually exclusive with `credentials`. [ credentials_file: ]