mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-25 05:34:05 -08:00
feat: add Google cloud roundtripper for remote write (#14346)
* feat: Google Auth for remote write Signed-off-by: Max Amin <maxamin@google.com> --------- Signed-off-by: Max Amin <maxamin@google.com>
This commit is contained in:
parent
79a0ba9d64
commit
84b819a69f
|
@ -37,6 +37,7 @@ import (
|
|||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/model/relabel"
|
||||
"github.com/prometheus/prometheus/storage/remote/azuread"
|
||||
"github.com/prometheus/prometheus/storage/remote/googleiam"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -1123,6 +1124,7 @@ type RemoteWriteConfig struct {
|
|||
MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"`
|
||||
SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"`
|
||||
AzureADConfig *azuread.AzureADConfig `yaml:"azuread,omitempty"`
|
||||
GoogleIAMConfig *googleiam.Config `yaml:"google_iam,omitempty"`
|
||||
}
|
||||
|
||||
// SetDirectory joins any relative file paths with dir.
|
||||
|
@ -1160,17 +1162,33 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
|
|||
return err
|
||||
}
|
||||
|
||||
httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil ||
|
||||
c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil
|
||||
return validateAuthConfigs(c)
|
||||
}
|
||||
|
||||
if httpClientConfigAuthEnabled && (c.SigV4Config != nil || c.AzureADConfig != nil) {
|
||||
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
|
||||
// validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured.
|
||||
func validateAuthConfigs(c *RemoteWriteConfig) error {
|
||||
var authConfigured []string
|
||||
if c.HTTPClientConfig.BasicAuth != nil {
|
||||
authConfigured = append(authConfigured, "basic_auth")
|
||||
}
|
||||
|
||||
if c.SigV4Config != nil && c.AzureADConfig != nil {
|
||||
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
|
||||
if c.HTTPClientConfig.Authorization != nil {
|
||||
authConfigured = append(authConfigured, "authorization")
|
||||
}
|
||||
if c.HTTPClientConfig.OAuth2 != nil {
|
||||
authConfigured = append(authConfigured, "oauth2")
|
||||
}
|
||||
if c.SigV4Config != nil {
|
||||
authConfigured = append(authConfigured, "sigv4")
|
||||
}
|
||||
if c.AzureADConfig != nil {
|
||||
authConfigured = append(authConfigured, "azuread")
|
||||
}
|
||||
if c.GoogleIAMConfig != nil {
|
||||
authConfigured = append(authConfigured, "google_iam")
|
||||
}
|
||||
if len(authConfigured) > 1 {
|
||||
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. Currently configured: %v", authConfigured)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -1189,7 +1207,7 @@ func validateHeadersForTracing(headers map[string]string) error {
|
|||
func validateHeaders(headers map[string]string) error {
|
||||
for header := range headers {
|
||||
if strings.ToLower(header) == "authorization" {
|
||||
return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter")
|
||||
return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter")
|
||||
}
|
||||
if _, ok := reservedHeaders[strings.ToLower(header)]; ok {
|
||||
return fmt.Errorf("%s is a reserved header. It must not be changed", header)
|
||||
|
|
|
@ -1826,7 +1826,7 @@ var expectedErrors = []struct {
|
|||
},
|
||||
{
|
||||
filename: "remote_write_authorization_header.bad.yml",
|
||||
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter`,
|
||||
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter`,
|
||||
},
|
||||
{
|
||||
filename: "remote_write_wrong_msg.bad.yml",
|
||||
|
|
|
@ -3401,8 +3401,8 @@ authorization:
|
|||
# It is mutually exclusive with `credentials`.
|
||||
[ credentials_file: <filename> ]
|
||||
|
||||
# Optionally configures AWS's Signature Verification 4 signing process to
|
||||
# sign requests. Cannot be set at the same time as basic_auth, authorization, or oauth2.
|
||||
# Optionally configures AWS's Signature Verification 4 signing process to sign requests.
|
||||
# Cannot be set at the same time as basic_auth, authorization, oauth2, azuread or google_iam.
|
||||
# To use the default credentials from the AWS SDK, use `sigv4: {}`.
|
||||
sigv4:
|
||||
# The AWS region. If blank, the region from the default credentials chain
|
||||
|
@ -3655,12 +3655,12 @@ sigv4:
|
|||
[ role_arn: <string> ]
|
||||
|
||||
# Optional OAuth 2.0 configuration.
|
||||
# Cannot be used at the same time as basic_auth, authorization, sigv4, or azuread.
|
||||
# Cannot be used at the same time as basic_auth, authorization, sigv4, azuread or google_iam.
|
||||
oauth2:
|
||||
[ <oauth2> ]
|
||||
|
||||
# Optional AzureAD configuration.
|
||||
# Cannot be used at the same time as basic_auth, authorization, oauth2, or sigv4.
|
||||
# Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or google_iam.
|
||||
azuread:
|
||||
# The Azure Cloud. Options are 'AzurePublic', 'AzureChina', or 'AzureGovernment'.
|
||||
[ cloud: <string> | default = AzurePublic ]
|
||||
|
@ -3680,6 +3680,14 @@ azuread:
|
|||
[ sdk:
|
||||
[ tenant_id: <string> ] ]
|
||||
|
||||
# WARNING: Remote write is NOT SUPPORTED by Google Cloud. This configuration is reserved for future use.
|
||||
# Optional Google Cloud Monitoring configuration.
|
||||
# Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or azuread.
|
||||
# To use the default credentials from the Google Cloud SDK, use `google_iam: {}`.
|
||||
google_iam:
|
||||
# Service account key with monitoring write permessions.
|
||||
credentials_file: <file_name>
|
||||
|
||||
# Configures the remote write request's TLS settings.
|
||||
tls_config:
|
||||
[ <tls_config> ]
|
||||
|
|
|
@ -26,7 +26,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
|
@ -51,7 +50,7 @@ const (
|
|||
func TestMain(m *testing.M) {
|
||||
// Enable experimental functions testing
|
||||
parser.EnableExperimentalFunctions = true
|
||||
goleak.VerifyTestMain(m)
|
||||
testutil.TolerantVerifyLeak(m)
|
||||
}
|
||||
|
||||
func TestQueryConcurrency(t *testing.T) {
|
||||
|
|
|
@ -32,7 +32,6 @@ import (
|
|||
"github.com/prometheus/common/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/atomic"
|
||||
"go.uber.org/goleak"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
|
@ -50,7 +49,7 @@ import (
|
|||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m)
|
||||
prom_testutil.TolerantVerifyLeak(m)
|
||||
}
|
||||
|
||||
func TestAlertingRule(t *testing.T) {
|
||||
|
|
|
@ -37,6 +37,7 @@ import (
|
|||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage/remote/azuread"
|
||||
"github.com/prometheus/prometheus/storage/remote/googleiam"
|
||||
)
|
||||
|
||||
const maxErrMsgLen = 1024
|
||||
|
@ -131,6 +132,7 @@ type ClientConfig struct {
|
|||
HTTPClientConfig config_util.HTTPClientConfig
|
||||
SigV4Config *sigv4.SigV4Config
|
||||
AzureADConfig *azuread.AzureADConfig
|
||||
GoogleIAMConfig *googleiam.Config
|
||||
Headers map[string]string
|
||||
RetryOnRateLimit bool
|
||||
WriteProtoMsg config.RemoteWriteProtoMsg
|
||||
|
@ -192,6 +194,13 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
|
|||
}
|
||||
}
|
||||
|
||||
if conf.GoogleIAMConfig != nil {
|
||||
t, err = googleiam.NewRoundTripper(conf.GoogleIAMConfig, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
writeProtoMsg := config.RemoteWriteProtoMsgV1
|
||||
if conf.WriteProtoMsg != "" {
|
||||
writeProtoMsg = conf.WriteProtoMsg
|
||||
|
|
54
storage/remote/googleiam/googleiam.go
Normal file
54
storage/remote/googleiam/googleiam.go
Normal file
|
@ -0,0 +1,54 @@
|
|||
// Copyright 2024 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package googleiam provides an http.RoundTripper that attaches an Google Cloud accessToken
|
||||
// to remote write requests.
|
||||
package googleiam
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/option"
|
||||
apihttp "google.golang.org/api/transport/http"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
CredentialsFile string `yaml:"credentials_file,omitempty"`
|
||||
}
|
||||
|
||||
// NewRoundTripper creates a round tripper that adds Google Cloud Monitoring authorization to calls
|
||||
// using either a credentials file or the default credentials.
|
||||
func NewRoundTripper(cfg *Config, next http.RoundTripper) (http.RoundTripper, error) {
|
||||
if next == nil {
|
||||
next = http.DefaultTransport
|
||||
}
|
||||
const scopes = "https://www.googleapis.com/auth/monitoring.write"
|
||||
ctx := context.Background()
|
||||
opts := []option.ClientOption{
|
||||
option.WithScopes(scopes),
|
||||
}
|
||||
if cfg.CredentialsFile != "" {
|
||||
opts = append(opts, option.WithCredentialsFile(cfg.CredentialsFile))
|
||||
} else {
|
||||
creds, err := google.FindDefaultCredentials(ctx, scopes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error finding default Google credentials: %w", err)
|
||||
}
|
||||
opts = append(opts, option.WithCredentials(creds))
|
||||
}
|
||||
|
||||
return apihttp.NewTransport(ctx, next, opts...)
|
||||
}
|
|
@ -176,6 +176,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
|
|||
HTTPClientConfig: rwConf.HTTPClientConfig,
|
||||
SigV4Config: rwConf.SigV4Config,
|
||||
AzureADConfig: rwConf.AzureADConfig,
|
||||
GoogleIAMConfig: rwConf.GoogleIAMConfig,
|
||||
Headers: rwConf.Headers,
|
||||
RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit,
|
||||
})
|
||||
|
|
|
@ -63,7 +63,10 @@ func TestMain(m *testing.M) {
|
|||
flag.Parse()
|
||||
defaultIsolationDisabled = !isolationEnabled
|
||||
|
||||
goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func1"), goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func2"))
|
||||
goleak.VerifyTestMain(m,
|
||||
goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func1"),
|
||||
goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func2"),
|
||||
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"))
|
||||
}
|
||||
|
||||
func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB) {
|
||||
|
|
Loading…
Reference in a new issue