Enable reading secrets from files

This only addresses the prometheus/prometheus portion of this issue.
Similar issues exist in other repositories under the prometheus
organization.

Closes: #8551

Signed-off-by: Marcelo E. Magallon <marcelo.magallon@grafana.com>

config/config.go

@@ -759,6 +759,33 @@ type MetadataConfig struct {
 	MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"`
 }
 
+// SigV4Config is the configuration for signing remote write requests with
+// AWS's SigV4 verification process. Empty values will be retrieved using the
+// AWS default credentials chain.
+type SigV4Config struct {
+	Region        string        `yaml:"region,omitempty"`
+	AccessKey     string        `yaml:"access_key,omitempty"`
+	SecretKey     config.Secret `yaml:"secret_key,omitempty"`
+	SecretKeyFile string        `yaml:"secret_key_file,omitempty"`
+	Profile       string        `yaml:"profile,omitempty"`
+	RoleARN       string        `yaml:"role_arn,omitempty"`
+}
+
+func (c *SigV4Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain SigV4Config
+	if err := unmarshal((*plain)(c)); err != nil {
+		return err
+	}
+
+	if len(c.SecretKey) == 0 && len(c.SecretKeyFile) != 0 {
+		if err := c.SecretKey.LoadFromFile(c.SecretKeyFile); err != nil {
+			return fmt.Errorf("cannot read sigv4 secret key: %w", err)
+		}
+	}
+
+	return nil
+}
+
 // RemoteReadConfig is the configuration for reading from remote storage.
 type RemoteReadConfig struct {
 	URL           *config.URL       `yaml:"url"`

discovery/aws/ec2.go

@@ -87,6 +87,7 @@ type EC2SDConfig struct {
 	Region          string         `yaml:"region"`
 	AccessKey       string         `yaml:"access_key,omitempty"`
 	SecretKey       config.Secret  `yaml:"secret_key,omitempty"`
+	SecretKeyFile   string         `yaml:"secret_key_file,omitempty"`
 	Profile         string         `yaml:"profile,omitempty"`
 	RoleARN         string         `yaml:"role_arn,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
@@ -110,6 +111,12 @@ func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err != nil {
 		return err
 	}
+	if len(c.SecretKey) == 0 && len(c.SecretKeyFile) != 0 {
+		err = c.SecretKey.LoadFromFile(c.SecretKeyFile)
+		if err != nil {
+			return fmt.Errorf("cannot read secret key: %w", err)
+		}
+	}
 	if c.Region == "" {
 		sess, err := session.NewSession()
 		if err != nil {

discovery/azure/azure.go

@@ -78,6 +78,7 @@ type SDConfig struct {
 	TenantID             string             `yaml:"tenant_id,omitempty"`
 	ClientID             string             `yaml:"client_id,omitempty"`
 	ClientSecret         config_util.Secret `yaml:"client_secret,omitempty"`
+	ClientSecretFile     string             `yaml:"client_secret_file,omitempty"`
 	RefreshInterval      model.Duration     `yaml:"refresh_interval,omitempty"`
 	AuthenticationMethod string             `yaml:"authentication_method,omitempty"`
 }
@@ -117,7 +118,13 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		if err = validateAuthParam(c.ClientID, "client_id"); err != nil {
 			return err
 		}
-		if err = validateAuthParam(string(c.ClientSecret), "client_secret"); err != nil {
+		if len(c.ClientSecret) == 0 && len(c.ClientSecretFile) != 0 {
+			err = c.ClientSecret.LoadFromFile(c.ClientSecretFile)
+			if err != nil {
+				return fmt.Errorf("cannot read client secret from file: %w", err)
+			}
+		}
+		if err := validateAuthParam(string(c.ClientSecret), "client_secret or client_secret_file"); err != nil {
 			return err
 		}
 	}

discovery/consul/consul.go

@@ -110,12 +110,14 @@ func init() {
 type SDConfig struct {
 	Server       string        `yaml:"server,omitempty"`
 	Token        config.Secret `yaml:"token,omitempty"`
+	TokenFile    string        `yaml:"token_file,omitempty"`
 	Datacenter   string        `yaml:"datacenter,omitempty"`
 	Namespace    string        `yaml:"namespace,omitempty"`
 	TagSeparator string        `yaml:"tag_separator,omitempty"`
 	Scheme       string        `yaml:"scheme,omitempty"`
 	Username     string        `yaml:"username,omitempty"`
 	Password     config.Secret `yaml:"password,omitempty"`
+	PasswordFile string        `yaml:"password_file,omitempty"`
 
 	// See https://www.consul.io/docs/internals/consensus.html#consistency-modes,
 	// stale reads are a lot cheaper and are a necessity if you have >5k targets.
@@ -162,6 +164,12 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if strings.TrimSpace(c.Server) == "" {
 		return errors.New("consul SD configuration requires a server address")
 	}
+	if len(c.Password) == 0 && len(c.PasswordFile) != 0 {
+		err = c.Password.LoadFromFile(c.PasswordFile)
+		if err != nil {
+			return fmt.Errorf("cannot read password: %w", err)
+		}
+	}
 	if c.Username != "" || c.Password != "" {
 		if c.HTTPClientConfig.BasicAuth != nil {
 			return errors.New("at most one of consul SD configuration username and password and basic auth can be configured")
@@ -171,7 +179,13 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			Password: c.Password,
 		}
 	}
-	if c.Token != "" && (c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil) {
+	tokenSet := false
+	if len(c.Token) != 0 && len(c.TokenFile) != 0 {
+		return errors.New("at most one of SD token or token_file can be configured")
+	} else if len(c.Token) != 0 || len(c.TokenFile) != 0 {
+		tokenSet = true
+	}
+	if tokenSet && (c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil) {
 		return errors.New("at most one of consul SD token, authorization, or oauth2 can be configured")
 	}
 	return c.HTTPClientConfig.Validate()
@@ -211,6 +225,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		Datacenter: conf.Datacenter,
 		Namespace:  conf.Namespace,
 		Token:      string(conf.Token),
+		TokenFile:  conf.TokenFile,
 		HttpClient: wrapper,
 	}
 	client, err := consul.NewClient(clientConf)

discovery/openstack/openstack.go

@@ -45,24 +45,26 @@ func init() {
 
 // SDConfig is the configuration for OpenStack based service discovery.
 type SDConfig struct {
-	IdentityEndpoint            string           `yaml:"identity_endpoint"`
+	IdentityEndpoint                string           `yaml:"identity_endpoint"`
-	Username                    string           `yaml:"username"`
+	Username                        string           `yaml:"username"`
-	UserID                      string           `yaml:"userid"`
+	UserID                          string           `yaml:"userid"`
-	Password                    config.Secret    `yaml:"password"`
+	Password                        config.Secret    `yaml:"password"`
-	ProjectName                 string           `yaml:"project_name"`
+	PasswordFile                    string           `yaml:"password_file"`
-	ProjectID                   string           `yaml:"project_id"`
+	ProjectName                     string           `yaml:"project_name"`
-	DomainName                  string           `yaml:"domain_name"`
+	ProjectID                       string           `yaml:"project_id"`
-	DomainID                    string           `yaml:"domain_id"`
+	DomainName                      string           `yaml:"domain_name"`
-	ApplicationCredentialName   string           `yaml:"application_credential_name"`
+	DomainID                        string           `yaml:"domain_id"`
-	ApplicationCredentialID     string           `yaml:"application_credential_id"`
+	ApplicationCredentialName       string           `yaml:"application_credential_name"`
-	ApplicationCredentialSecret config.Secret    `yaml:"application_credential_secret"`
+	ApplicationCredentialID         string           `yaml:"application_credential_id"`
-	Role                        Role             `yaml:"role"`
+	ApplicationCredentialSecret     config.Secret    `yaml:"application_credential_secret"`
-	Region                      string           `yaml:"region"`
+	ApplicationCredentialSecretFile string           `yaml:"application_credential_secret_file"`
-	RefreshInterval             model.Duration   `yaml:"refresh_interval"`
+	Role                            Role             `yaml:"role"`
-	Port                        int              `yaml:"port"`
+	Region                          string           `yaml:"region"`
-	AllTenants                  bool             `yaml:"all_tenants,omitempty"`
+	RefreshInterval                 model.Duration   `yaml:"refresh_interval"`
-	TLSConfig                   config.TLSConfig `yaml:"tls_config,omitempty"`
+	Port                            int              `yaml:"port"`
-	Availability                string           `yaml:"availability,omitempty"`
+	AllTenants                      bool             `yaml:"all_tenants,omitempty"`
+	TLSConfig                       config.TLSConfig `yaml:"tls_config,omitempty"`
+	Availability                    string           `yaml:"availability,omitempty"`
 }
 
 // Name returns the name of the Config.
@@ -113,6 +115,20 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
+	if len(c.Password) == 0 && len(c.PasswordFile) != 0 {
+		err = c.Password.LoadFromFile(c.PasswordFile)
+		if err != nil {
+			return fmt.Errorf("cannot read password: %w", err)
+		}
+	}
+
+	if len(c.ApplicationCredentialSecret) == 0 && len(c.ApplicationCredentialSecretFile) != 0 {
+		err = c.ApplicationCredentialSecret.LoadFromFile(c.ApplicationCredentialSecretFile)
+		if err != nil {
+			return fmt.Errorf("cannot read application credential secret: %w", err)
+		}
+	}
+
 	switch c.Availability {
 	case "public", "internal", "admin":
 	default:

docs/configuration/configuration.md

@@ -416,9 +416,15 @@ subscription_id: <string>
 [ tenant_id: <string> ]
 # Optional client ID. Only required with authentication_method OAuth.
 [ client_id: <string> ]
+
 # Optional client secret. Only required with authentication_method OAuth.
+# It is given preference over `client_secret_file`.
 [ client_secret: <secret> ]
 
+# Path to file containing the client secret.
+# It is used if `client_secret` is not set.
+[ client_secret_file: <string> ]
+
 # Refresh interval to re-read the instance list.
 [ refresh_interval: <duration> | default = 300s ]
 
@@ -451,7 +457,15 @@ The following meta labels are available on targets during [relabeling](#relabel_
 # The information to access the Consul API. It is to be defined
 # as the Consul documentation requires.
 [ server: <host> | default = "localhost:8500" ]
+
+# Token to use when accessing the API.
+# It is given preference over `token_file`.
 [ token: <secret> ]
+
+# Path to file containing the token to use when accessing the API.
+# It is used if `token` is not set.
+[ token_file: <string> ]
+
 [ datacenter: <string> ]
 # Namespaces are only supported in Consul Enterprise.
 [ namespace: <string> ]
@@ -459,6 +473,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
 # The username and password fields are deprecated in favor of the basic_auth configuration.
 [ username: <string> ]
 [ password: <secret> ]
+[ password_file: <string> ]
 
 # A list of services for which targets are retrieved. If omitted, all services
 # are scraped.
@@ -936,7 +951,15 @@ See below for the configuration options for EC2 discovery:
 # The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID`
 # and `AWS_SECRET_ACCESS_KEY` are used.
 [ access_key: <string> ]
+
+# Secret key to use when accessing the API.
+# It is given preference over `secret_key_file`.
 [ secret_key: <secret> ]
+
+# Path to file containing the secret key to use when accessing the API.
+# It is used if `secret_key` is not set.
+[ secret_key_file: <string> ]
+
 # Named AWS profile used to connect to the API.
 [ profile: <string> ]
 
@@ -1029,7 +1052,10 @@ region: <string>
 
 # password for the Identity V2 and V3 APIs. Consult with your provider's
 # control panel to discover your account's preferred method of authentication.
+# You can alternatively provide a path to a file containing the # password.
+# If both are provided, `password` is used.
 [ password: <secret> ]
+[ password_file: <string> ]
 
 # At most one of domain_id and domain_name must be provided if using username
 # with Identity V3. Otherwise, either are optional.
@@ -1051,8 +1077,11 @@ region: <string>
 [ application_credential_id: <string> ]
 
 # The application_credential_secret field is required if using an application
-# credential to authenticate.
+# credential to authenticate. Alternatively you can provide a path to the file
+# containing the secret. If both are provided, `application_credential_secret`
+# is used.
 [ application_credential_secret: <secret> ]
+[ application_credential_secret_file: <string> ]
 
 # Whether the service discovery should list all instances for all projects.
 # It is only relevant for the 'instance' role and usually requires admin permissions.
@@ -1772,7 +1801,15 @@ See below for the configuration options for Lightsail discovery:
 # The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID`
 # and `AWS_SECRET_ACCESS_KEY` are used.
 [ access_key: <string> ]
+
+# Secret key to use when accessing the API.
+# It is given preference over `secret_key_file`.
 [ secret_key: <secret> ]
+
+# Path to file containing the secret key to use when accessing the API.
+# It is used if `secret_key` is not set.
+[ secret_key_file: <string> ]
+
 # Named AWS profile used to connect to the API.
 [ profile: <string> ]
 
@@ -2590,8 +2627,15 @@ sigv4:
   # The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID`
   # and `AWS_SECRET_ACCESS_KEY` are used.
   [ access_key: <string> ]
+
+  # Secret key to use when accessing the API.
+  # It is given preference over `secret_key_file`.
   [ secret_key: <secret> ]
 
+  # Path to file containing the secret key to use when accessing the API.
+  # It is used if `secret_key` is not set.
+  [ secret_key_file: <string> ]
+
   # Named AWS profile used to authenticate.
   [ profile: <string> ]
 

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/prometheus/alertmanager v0.23.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.31.1
+	github.com/prometheus/common v0.31.2-0.20211011203104-9789762a2ddb
 	github.com/prometheus/common/sigv4 v0.1.0
 	github.com/prometheus/exporter-toolkit v0.6.1
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44

go.sum

@@ -1141,8 +1141,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.31.1 h1:d18hG4PkHnNAKNMOmFuXFaiY8Us0nird/2m60uS1AMs=
+github.com/prometheus/common v0.31.2-0.20211011203104-9789762a2ddb h1:nDUL0g/BSYon1707Ums2YQG50fvMgV2D8otbQZHNnEs=
-github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.31.2-0.20211011203104-9789762a2ddb/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
 github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
 github.com/prometheus/exporter-toolkit v0.6.1 h1:Aqk75wQD92N9CqmTlZwjKwq6272nOGrWIbc8Z7+xQO0=