mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Switch to common/sigv4

Browse source 
Signed-off-by: Levi Harrison <git@leviharrison.dev>
This commit is contained in:
Levi Harrison 2021-08-26 09:37:19 -04:00
parent 4323aa00a7
commit bd57cd395e
No known key found for this signature in database
GPG key ID: 23A7BDEAE860B57D
6 changed files with 9 additions and 245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
config/config.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/sigv4"
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/discovery"
@ -666,7 +667,7 @@ type RemoteWriteConfig struct {
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
QueueConfig QueueConfig `yaml:"queue_config,omitempty"`
MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"`
SigV4Config *SigV4Config `yaml:"sigv4,omitempty"`
SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
@ -758,17 +759,6 @@ type MetadataConfig struct {
MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"`
}
// SigV4Config is the configuration for signing remote write requests with
// AWS's SigV4 verification process. Empty values will be retrieved using the
// AWS default credentials chain.
type SigV4Config struct {
Region string `yaml:"region,omitempty"`
AccessKey string `yaml:"access_key,omitempty"`
SecretKey config.Secret `yaml:"secret_key,omitempty"`
Profile string `yaml:"profile,omitempty"`
RoleARN string `yaml:"role_arn,omitempty"`
}
// RemoteReadConfig is the configuration for reading from remote storage.
type RemoteReadConfig struct {
URL *config.URL `yaml:"url"`

1
go.mod
View file

@ -47,6 +47,7 @@ require (
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.30.0
github.com/prometheus/common/sigv4 v0.1.0
github.com/prometheus/exporter-toolkit v0.6.1
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749

3
go.sum
View file

@ -180,6 +180,7 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.40.10 h1:h+xUINuuH/9CwxE7O8mAuW7Aj9E5agfE9jQ3DrJsnA8=
github.com/aws/aws-sdk-go v1.40.10/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
@ -1122,6 +1123,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug=
github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg=
github.com/prometheus/exporter-toolkit v0.6.1 h1:Aqk75wQD92N9CqmTlZwjKwq6272nOGrWIbc8Z7+xQO0=
github.com/prometheus/exporter-toolkit v0.6.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g=

6
storage/remote/client.go
View file

@ -33,9 +33,9 @@ import (
"github.com/prometheus/client_golang/prometheus"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/sigv4"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/prompb"
)
@ -97,7 +97,7 @@ type ClientConfig struct {
URL *config_util.URL
Timeout model.Duration
HTTPClientConfig config_util.HTTPClientConfig
SigV4Config *config.SigV4Config
SigV4Config *sigv4.SigV4Config
Headers map[string]string
RetryOnRateLimit bool
}
@ -143,7 +143,7 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
t := httpClient.Transport
if conf.SigV4Config != nil {
t, err = newSigV4RoundTripper(conf.SigV4Config, httpClient.Transport)
t, err = sigv4.NewSigV4RoundTripper(conf.SigV4Config, httpClient.Transport)
if err != nil {
return nil, err
}

138
storage/remote/sigv4.go
View file

@ -1,138 +0,0 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/textproto"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/session"
signer "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/prometheus/prometheus/config"
)
var sigv4HeaderDenylist = []string{
"uber-trace-id",
}
type sigV4RoundTripper struct {
region string
next http.RoundTripper
pool sync.Pool
signer *signer.Signer
}
// newSigV4RoundTripper returns a new http.RoundTripper that will sign requests
// using Amazon's Signature Verification V4 signing procedure. The request will
// then be handed off to the next RoundTripper provided by next. If next is nil,
// http.DefaultTransport will be used.
//
// Credentials for signing are retrieved using the the default AWS credential
// chain. If credentials cannot be found, an error will be returned.
func newSigV4RoundTripper(cfg *config.SigV4Config, next http.RoundTripper) (http.RoundTripper, error) {
if next == nil {
next = http.DefaultTransport
}
creds := credentials.NewStaticCredentials(cfg.AccessKey, string(cfg.SecretKey), "")
if cfg.AccessKey == "" && cfg.SecretKey == "" {
creds = nil
}
sess, err := session.NewSessionWithOptions(session.Options{
Config: aws.Config{
Region: aws.String(cfg.Region),
Credentials: creds,
},
Profile: cfg.Profile,
})
if err != nil {
return nil, fmt.Errorf("could not create new AWS session: %w", err)
}
if _, err := sess.Config.Credentials.Get(); err != nil {
return nil, fmt.Errorf("could not get SigV4 credentials: %w", err)
}
if aws.StringValue(sess.Config.Region) == "" {
return nil, fmt.Errorf("region not configured in sigv4 or in default credentials chain")
}
signerCreds := sess.Config.Credentials
if cfg.RoleARN != "" {
signerCreds = stscreds.NewCredentials(sess, cfg.RoleARN)
}
rt := &sigV4RoundTripper{
region: cfg.Region,
next: next,
signer: signer.NewSigner(signerCreds),
}
rt.pool.New = rt.newBuf
return rt, nil
}
func (rt *sigV4RoundTripper) newBuf() interface{} {
return bytes.NewBuffer(make([]byte, 0, 1024))
}
func (rt *sigV4RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// rt.signer.Sign needs a seekable body, so we replace the body with a
// buffered reader filled with the contents of original body.
buf := rt.pool.Get().(*bytes.Buffer)
defer func() {
buf.Reset()
rt.pool.Put(buf)
}()
if _, err := io.Copy(buf, req.Body); err != nil {
return nil, err
}
// Close the original body since we don't need it anymore.
_ = req.Body.Close()
// Ensure our seeker is back at the start of the buffer once we return.
var seeker io.ReadSeeker = bytes.NewReader(buf.Bytes())
defer func() {
_, _ = seeker.Seek(0, io.SeekStart)
}()
req.Body = ioutil.NopCloser(seeker)
// Clone the request and trim out headers that we don't want to sign.
signReq := req.Clone(req.Context())
for _, header := range sigv4HeaderDenylist {
signReq.Header.Del(header)
}
headers, err := rt.signer.Sign(signReq, seeker, "aps", rt.region, time.Now().UTC())
if err != nil {
return nil, fmt.Errorf("failed to sign request: %w", err)
}
// Copy over signed headers. Authorization header is not returned by
// rt.signer.Sign and needs to be copied separately.
for k, v := range headers {
req.Header[textproto.CanonicalMIMEHeaderKey(k)] = v
}
req.Header.Set("Authorization", signReq.Header.Get("Authorization"))
return rt.next.RoundTrip(req)
}

92
storage/remote/sigv4_test.go
View file

@ -1,92 +0,0 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"net/http"
"os"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
signer "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/stretchr/testify/require"
)
func TestSigV4_Inferred_Region(t *testing.T) {
os.Setenv("AWS_ACCESS_KEY_ID", "secret")
os.Setenv("AWS_SECRET_ACCESS_KEY", "token")
os.Setenv("AWS_REGION", "us-west-2")
sess, err := session.NewSession(&aws.Config{
// Setting to an empty string to demostrate the default value from the yaml
// won't override the environment's region.
Region: aws.String(""),
})
require.NoError(t, err)
_, err = sess.Config.Credentials.Get()
require.NoError(t, err)
require.NotNil(t, sess.Config.Region)
require.Equal(t, "us-west-2", *sess.Config.Region)
}
func TestSigV4RoundTripper(t *testing.T) {
var gotReq *http.Request
rt := &sigV4RoundTripper{
region: "us-east-2",
next: promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
gotReq = req
return &http.Response{StatusCode: http.StatusOK}, nil
}),
signer: signer.NewSigner(credentials.NewStaticCredentials(
"test-id",
"secret",
"token",
)),
}
rt.pool.New = rt.newBuf
cli := &http.Client{Transport: rt}
req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!"))
require.NoError(t, err)
_, err = cli.Do(req)
require.NoError(t, err)
require.NotNil(t, gotReq)
origReq := gotReq
require.NotEmpty(t, origReq.Header.Get("Authorization"))
require.NotEmpty(t, origReq.Header.Get("X-Amz-Date"))
// Perform the same request but with a header that shouldn't included in the
// signature; validate that the Authorization signature matches.
t.Run("Ignored Headers", func(t *testing.T) {
req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!"))
require.NoError(t, err)
req.Header.Add("Uber-Trace-Id", "some-trace-id")
_, err = cli.Do(req)
require.NoError(t, err)
require.NotNil(t, gotReq)
require.Equal(t, origReq.Header.Get("Authorization"), gotReq.Header.Get("Authorization"))
})
}