remote_write: allow passing along custom HTTP headers (#8416)

* remote_write: allow passing along custom HTTP headers

Signed-off-by: Nandor Kracser <bonifaido@gmail.com>

* add warning

Signed-off-by: Nandor Kracser <bonifaido@gmail.com>

* remote_write: add header valadtion

Signed-off-by: Nandor Kracser <bonifaido@gmail.com>

* extend tests for bad remote write headers

Signed-off-by: Nandor Kracser <bonifaido@gmail.com>

* remote_write: add note about the authorization header

Signed-off-by: Nandor Kracser <bonifaido@gmail.com>

config/config.go

@@ -34,6 +34,21 @@ import (
 
 var (
 	patRulePath         = regexp.MustCompile(`^[^*]*(\*[^/]*)?$`)
+	unchangeableHeaders = map[string]struct{}{
+		// NOTE: authorization is checked specially,
+		// see RemoteWriteConfig.UnmarshalYAML.
+		// "authorization":                  {},
+		"host":                              {},
+		"content-encoding":                  {},
+		"content-type":                      {},
+		"x-prometheus-remote-write-version": {},
+		"user-agent":                        {},
+		"connection":                        {},
+		"keep-alive":                        {},
+		"proxy-authenticate":                {},
+		"proxy-authorization":               {},
+		"www-authenticate":                  {},
+	}
 )
 
 // Load parses the YAML input s into a Config.
@@ -570,6 +585,7 @@ func CheckTargetAddress(address model.LabelValue) error {
 type RemoteWriteConfig struct {
 	URL                 *config.URL       `yaml:"url"`
 	RemoteTimeout       model.Duration    `yaml:"remote_timeout,omitempty"`
+	Headers             map[string]string `yaml:"headers,omitempty"`
 	WriteRelabelConfigs []*relabel.Config `yaml:"write_relabel_configs,omitempty"`
 	Name                string            `yaml:"name,omitempty"`
 
@@ -600,6 +616,14 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
 			return errors.New("empty or null relabeling rule in remote write config")
 		}
 	}
+	for header := range c.Headers {
+		if strings.ToLower(header) == "authorization" {
+			return errors.New("authorization header must be changed via the basic_auth, bearer_token, or bearer_token_file parameter")
+		}
+		if _, ok := unchangeableHeaders[strings.ToLower(header)]; ok {
+			return errors.Errorf("%s is an unchangeable header", header)
+		}
+	}
 
 	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
 	// We cannot make it a pointer as the parser panics for inlined pointer structs.

config/config_test.go

@@ -102,6 +102,7 @@ var expectedConf = &Config{
 					KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 				},
 			},
+			Headers: map[string]string{"name": "value"},
 		},
 	},
 
@@ -937,6 +938,12 @@ var expectedErrors = []struct {
 	}, {
 		filename: "remote_read_url_missing.bad.yml",
 		errMsg:   `url for remote_read is empty`,
+	}, {
+		filename: "remote_write_header.bad.yml",
+		errMsg:   `x-prometheus-remote-write-version is an unchangeable header`,
+	}, {
+		filename: "remote_write_authorization_header.bad.yml",
+		errMsg:   `authorization header must be changed via the basic_auth, bearer_token, or bearer_token_file parameter`,
 	}, {
 		filename: "remote_write_url_missing.bad.yml",
 		errMsg:   `url for remote_write is empty`,

config/testdata/conf.good.yml

@@ -24,6 +24,8 @@ remote_write:
     tls_config:
       cert_file: valid_cert_file
       key_file: valid_key_file
+    headers:
+      name: value
 
 remote_read:
   - url: http://remote1/read

config/testdata/remote_write_authorization_header.bad.yml

@@ -0,0 +1,5 @@
+remote_write:
+  - url: localhost:9090
+    name: queue1
+    headers:
+      "authorization": "Basic YWxhZGRpbjpvcGVuc2VzYW1l"

config/testdata/remote_write_header.bad.yml

@@ -0,0 +1,5 @@
+remote_write:
+  - url: localhost:9090
+    name: queue1
+    headers:
+      "x-prometheus-remote-write-version": "somehack"

docs/configuration/configuration.md

@@ -1719,6 +1719,11 @@ url: <string>
 # Timeout for requests to the remote write endpoint.
 [ remote_timeout: <duration> | default = 30s ]
 
+# Custom HTTP headers to be sent along with each remote write request.
+# Be aware that headers that are set by Prometheus itself can't be overwritten.
+headers:
+  [ <string>: <string> ... ]
+
 # List of remote write relabel configurations.
 write_relabel_configs:
   [ - <relabel_config> ... ]

storage/remote/client.go

@@ -83,6 +83,7 @@ type Client struct {
 	url        *config_util.URL
 	Client     *http.Client
 	timeout    time.Duration
+	headers    map[string]string
 
 	readQueries         prometheus.Gauge
 	readQueriesTotal    *prometheus.CounterVec
@@ -94,6 +95,7 @@ type ClientConfig struct {
 	URL              *config_util.URL
 	Timeout          model.Duration
 	HTTPClientConfig config_util.HTTPClientConfig
+	Headers          map[string]string
 }
 
 // ReadClient uses the SAMPLES method of remote read to read series samples from remote server.
@@ -142,6 +144,7 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
 		url:        conf.URL,
 		Client:     httpClient,
 		timeout:    time.Duration(conf.Timeout),
+		headers:    conf.Headers,
 	}, nil
 }
 
@@ -158,6 +161,9 @@ func (c *Client) Store(ctx context.Context, req []byte) error {
 		// recoverable.
 		return err
 	}
+	for k, v := range c.headers {
+		httpReq.Header.Set(k, v)
+	}
 	httpReq.Header.Add("Content-Encoding", "snappy")
 	httpReq.Header.Set("Content-Type", "application/x-protobuf")
 	httpReq.Header.Set("User-Agent", UserAgent)

storage/remote/write.go

@@ -134,6 +134,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 			URL:              rwConf.URL,
 			Timeout:          rwConf.RemoteTimeout,
 			HTTPClientConfig: rwConf.HTTPClientConfig,
+			Headers:          rwConf.Headers,
 		})
 		if err != nil {
 			return err