Invert service discovery dependencies (#7701)

This also fixes a bug in query_log_file, which now is relative to the config file like all other paths.

Signed-off-by: Andy Bursavich <abursavich@gmail.com>

cmd/prometheus/main.go

@@ -53,7 +53,6 @@ import (
 	promlogflag "github.com/prometheus/common/promlog/flag"
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
-	sd_config "github.com/prometheus/prometheus/discovery/config"
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/logging"
@@ -67,6 +66,8 @@ import (
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/prometheus/prometheus/web"
+
+	_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
 )
 
 var (
@@ -466,9 +467,9 @@ func main() {
 		}, {
 			name: "scrape_sd",
 			reloader: func(cfg *config.Config) error {
-				c := make(map[string]sd_config.ServiceDiscoveryConfig)
+				c := make(map[string]discovery.Configs)
 				for _, v := range cfg.ScrapeConfigs {
-					c[v.JobName] = v.ServiceDiscoveryConfig
+					c[v.JobName] = v.ServiceDiscoveryConfigs
 				}
 				return discoveryManagerScrape.ApplyConfig(c)
 			},
@@ -478,9 +479,9 @@ func main() {
 		}, {
 			name: "notify_sd",
 			reloader: func(cfg *config.Config) error {
-				c := make(map[string]sd_config.ServiceDiscoveryConfig)
+				c := make(map[string]discovery.Configs)
 				for k, v := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
-					c[k] = v.ServiceDiscoveryConfig
+					c[k] = v.ServiceDiscoveryConfigs
 				}
 				return discoveryManagerNotify.ApplyConfig(c)
 			},

cmd/promtool/main.go

@@ -40,7 +40,11 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/discovery/file"
+	"github.com/prometheus/prometheus/discovery/kubernetes"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
+
+	_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
 )
 
 func main() {
@@ -263,14 +267,14 @@ func checkConfig(filename string) ([]string, error) {
 			return nil, err
 		}
 
-		for _, kd := range scfg.ServiceDiscoveryConfig.KubernetesSDConfigs {
+		for _, c := range scfg.ServiceDiscoveryConfigs {
-			if err := checkTLSConfig(kd.HTTPClientConfig.TLSConfig); err != nil {
+			switch c := c.(type) {
+			case *kubernetes.SDConfig:
+				if err := checkTLSConfig(c.HTTPClientConfig.TLSConfig); err != nil {
 					return nil, err
 				}
-		}
+			case *file.SDConfig:
-
+				for _, file := range c.Files {
-		for _, filesd := range scfg.ServiceDiscoveryConfig.FileSDConfigs {
-			for _, file := range filesd.Files {
 					files, err := filepath.Glob(file)
 					if err != nil {
 						return nil, err
@@ -284,6 +288,7 @@ func checkConfig(filename string) ([]string, error) {
 				}
 			}
 		}
+	}
 
 	return ruleFiles, nil
 }

config/config.go

@@ -23,11 +23,11 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	yaml "gopkg.in/yaml.v2"
 
-	sd_config "github.com/prometheus/prometheus/discovery/config"
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/relabel"
 )
@@ -48,7 +48,6 @@ func Load(s string) (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	cfg.original = s
 	return cfg, nil
 }
 
@@ -62,7 +61,7 @@ func LoadFile(filename string) (*Config, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "parsing YAML file %s", filename)
 	}
-	resolveFilepaths(filepath.Dir(filename), cfg)
+	cfg.SetDirectory(filepath.Dir(filename))
 	return cfg, nil
 }
 
@@ -137,80 +136,23 @@ type Config struct {
 
 	RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
 	RemoteReadConfigs  []*RemoteReadConfig  `yaml:"remote_read,omitempty"`
-
-	// original is the input from which the config was parsed.
-	original string
 }
 
-// resolveFilepaths joins all relative paths in a configuration
+// SetDirectory joins any relative file paths with dir.
-// with a given base directory.
+func (c *Config) SetDirectory(dir string) {
-func resolveFilepaths(baseDir string, cfg *Config) {
+	c.GlobalConfig.SetDirectory(dir)
-	join := func(fp string) string {
+	c.AlertingConfig.SetDirectory(dir)
-		if len(fp) > 0 && !filepath.IsAbs(fp) {
+	for i, file := range c.RuleFiles {
-			fp = filepath.Join(baseDir, fp)
+		c.RuleFiles[i] = config.JoinDir(dir, file)
 	}
-		return fp
+	for _, c := range c.ScrapeConfigs {
+		c.SetDirectory(dir)
 	}
-
+	for _, c := range c.RemoteWriteConfigs {
-	for i, rf := range cfg.RuleFiles {
+		c.SetDirectory(dir)
-		cfg.RuleFiles[i] = join(rf)
 	}
-
+	for _, c := range c.RemoteReadConfigs {
-	tlsPaths := func(cfg *config_util.TLSConfig) {
+		c.SetDirectory(dir)
-		cfg.CAFile = join(cfg.CAFile)
-		cfg.CertFile = join(cfg.CertFile)
-		cfg.KeyFile = join(cfg.KeyFile)
-	}
-	clientPaths := func(scfg *config_util.HTTPClientConfig) {
-		if scfg.BasicAuth != nil {
-			scfg.BasicAuth.PasswordFile = join(scfg.BasicAuth.PasswordFile)
-		}
-		scfg.BearerTokenFile = join(scfg.BearerTokenFile)
-		tlsPaths(&scfg.TLSConfig)
-	}
-	sdPaths := func(cfg *sd_config.ServiceDiscoveryConfig) {
-		for _, kcfg := range cfg.KubernetesSDConfigs {
-			clientPaths(&kcfg.HTTPClientConfig)
-		}
-		for _, mcfg := range cfg.MarathonSDConfigs {
-			mcfg.AuthTokenFile = join(mcfg.AuthTokenFile)
-			clientPaths(&mcfg.HTTPClientConfig)
-		}
-		for _, consulcfg := range cfg.ConsulSDConfigs {
-			tlsPaths(&consulcfg.TLSConfig)
-		}
-		for _, digitaloceancfg := range cfg.DigitalOceanSDConfigs {
-			clientPaths(&digitaloceancfg.HTTPClientConfig)
-		}
-		for _, dockerswarmcfg := range cfg.DockerSwarmSDConfigs {
-			clientPaths(&dockerswarmcfg.HTTPClientConfig)
-		}
-		for _, cfg := range cfg.OpenstackSDConfigs {
-			tlsPaths(&cfg.TLSConfig)
-		}
-		for _, cfg := range cfg.TritonSDConfigs {
-			tlsPaths(&cfg.TLSConfig)
-		}
-		for _, filecfg := range cfg.FileSDConfigs {
-			for i, fn := range filecfg.Files {
-				filecfg.Files[i] = join(fn)
-			}
-		}
-	}
-
-	for _, cfg := range cfg.ScrapeConfigs {
-		clientPaths(&cfg.HTTPClientConfig)
-		sdPaths(&cfg.ServiceDiscoveryConfig)
-	}
-	for _, cfg := range cfg.AlertingConfig.AlertmanagerConfigs {
-		clientPaths(&cfg.HTTPClientConfig)
-		sdPaths(&cfg.ServiceDiscoveryConfig)
-	}
-	for _, cfg := range cfg.RemoteReadConfigs {
-		clientPaths(&cfg.HTTPClientConfig)
-	}
-	for _, cfg := range cfg.RemoteWriteConfigs {
-		clientPaths(&cfg.HTTPClientConfig)
 	}
 }
 
@@ -311,6 +253,11 @@ type GlobalConfig struct {
 	ExternalLabels labels.Labels `yaml:"external_labels,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *GlobalConfig) SetDirectory(dir string) {
+	c.QueryLogFile = config.JoinDir(dir, c.QueryLogFile)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// Create a clean global config as the previous one was already populated
@@ -388,8 +335,8 @@ type ScrapeConfig struct {
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
 
-	ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"`
+	ServiceDiscoveryConfigs discovery.Configs       `yaml:"-"`
-	HTTPClientConfig       config_util.HTTPClientConfig     `yaml:",inline"`
+	HTTPClientConfig        config.HTTPClientConfig `yaml:",inline"`
 
 	// List of target relabel configurations.
 	RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
@@ -397,12 +344,16 @@ type ScrapeConfig struct {
 	MetricRelabelConfigs []*relabel.Config `yaml:"metric_relabel_configs,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *ScrapeConfig) SetDirectory(dir string) {
+	c.ServiceDiscoveryConfigs.SetDirectory(dir)
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultScrapeConfig
-	type plain ScrapeConfig
+	if err := discovery.UnmarshalYAMLWithInlineConfigs(c, unmarshal); err != nil {
-	err := unmarshal((*plain)(c))
-	if err != nil {
 		return err
 	}
 	if len(c.JobName) == 0 {
@@ -416,23 +367,12 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
-	// The UnmarshalYAML method of ServiceDiscoveryConfig is not being called because it's not a pointer.
-	// We cannot make it a pointer as the parser panics for inlined pointer structs.
-	// Thus we just do its validation here.
-	if err := c.ServiceDiscoveryConfig.Validate(); err != nil {
-		return err
-	}
-
 	// Check for users putting URLs in target groups.
 	if len(c.RelabelConfigs) == 0 {
-		for _, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
+		if err := checkStaticTargets(c.ServiceDiscoveryConfigs); err != nil {
-			for _, t := range tg.Targets {
-				if err := CheckTargetAddress(t[model.AddressLabel]); err != nil {
 			return err
 		}
 	}
-		}
-	}
 
 	for _, rlcfg := range c.RelabelConfigs {
 		if rlcfg == nil {
@@ -445,13 +385,12 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		}
 	}
 
-	// Add index to the static config target groups for unique identification
+	return nil
-	// within scrape pool.
-	for i, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
-		tg.Source = fmt.Sprintf("%d", i)
 }
 
-	return nil
+// MarshalYAML implements the yaml.Marshaler interface.
+func (c *ScrapeConfig) MarshalYAML() (interface{}, error) {
+	return discovery.MarshalYAMLWithInlineConfigs(c)
 }
 
 // AlertingConfig configures alerting and alertmanager related configs.
@@ -460,6 +399,13 @@ type AlertingConfig struct {
 	AlertmanagerConfigs AlertmanagerConfigs `yaml:"alertmanagers,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *AlertingConfig) SetDirectory(dir string) {
+	for _, c := range c.AlertmanagerConfigs {
+		c.SetDirectory(dir)
+	}
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// Create a clean global config as the previous one was already populated
@@ -529,8 +475,8 @@ type AlertmanagerConfig struct {
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
 
-	ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"`
+	ServiceDiscoveryConfigs discovery.Configs       `yaml:"-"`
-	HTTPClientConfig       config_util.HTTPClientConfig     `yaml:",inline"`
+	HTTPClientConfig        config.HTTPClientConfig `yaml:",inline"`
 
 	// The URL scheme to use when talking to Alertmanagers.
 	Scheme string `yaml:"scheme,omitempty"`
@@ -546,11 +492,16 @@ type AlertmanagerConfig struct {
 	RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *AlertmanagerConfig) SetDirectory(dir string) {
+	c.ServiceDiscoveryConfigs.SetDirectory(dir)
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultAlertmanagerConfig
-	type plain AlertmanagerConfig
+	if err := discovery.UnmarshalYAMLWithInlineConfigs(c, unmarshal); err != nil {
-	if err := unmarshal((*plain)(c)); err != nil {
 		return err
 	}
 
@@ -561,23 +512,12 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
 		return err
 	}
 
-	// The UnmarshalYAML method of ServiceDiscoveryConfig is not being called because it's not a pointer.
-	// We cannot make it a pointer as the parser panics for inlined pointer structs.
-	// Thus we just do its validation here.
-	if err := c.ServiceDiscoveryConfig.Validate(); err != nil {
-		return err
-	}
-
 	// Check for users putting URLs in target groups.
 	if len(c.RelabelConfigs) == 0 {
-		for _, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
+		if err := checkStaticTargets(c.ServiceDiscoveryConfigs); err != nil {
-			for _, t := range tg.Targets {
-				if err := CheckTargetAddress(t[model.AddressLabel]); err != nil {
 			return err
 		}
 	}
-		}
-	}
 
 	for _, rlcfg := range c.RelabelConfigs {
 		if rlcfg == nil {
@@ -585,12 +525,28 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
 		}
 	}
 
-	// Add index to the static config target groups for unique identification
+	return nil
-	// within scrape pool.
-	for i, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
-		tg.Source = fmt.Sprintf("%d", i)
 }
 
+// MarshalYAML implements the yaml.Marshaler interface.
+func (c *AlertmanagerConfig) MarshalYAML() (interface{}, error) {
+	return discovery.MarshalYAMLWithInlineConfigs(c)
+}
+
+func checkStaticTargets(configs discovery.Configs) error {
+	for _, cfg := range configs {
+		sc, ok := cfg.(discovery.StaticConfig)
+		if !ok {
+			continue
+		}
+		for _, tg := range sc {
+			for _, t := range tg.Targets {
+				if err := CheckTargetAddress(t[model.AddressLabel]); err != nil {
+					return err
+				}
+			}
+		}
+	}
 	return nil
 }
 
@@ -603,31 +559,24 @@ func CheckTargetAddress(address model.LabelValue) error {
 	return nil
 }
 
-// ClientCert contains client cert credentials.
-type ClientCert struct {
-	Cert string             `yaml:"cert"`
-	Key  config_util.Secret `yaml:"key"`
-}
-
-// FileSDConfig is the configuration for file based discovery.
-type FileSDConfig struct {
-	Files           []string       `yaml:"files"`
-	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
-}
-
 // RemoteWriteConfig is the configuration for writing to remote storage.
 type RemoteWriteConfig struct {
-	URL                 *config_util.URL  `yaml:"url"`
+	URL                 *config.URL       `yaml:"url"`
 	RemoteTimeout       model.Duration    `yaml:"remote_timeout,omitempty"`
 	WriteRelabelConfigs []*relabel.Config `yaml:"write_relabel_configs,omitempty"`
 	Name                string            `yaml:"name,omitempty"`
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
-	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 	QueueConfig      QueueConfig             `yaml:"queue_config,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *RemoteWriteConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultRemoteWriteConfig
@@ -676,20 +625,25 @@ type QueueConfig struct {
 
 // RemoteReadConfig is the configuration for reading from remote storage.
 type RemoteReadConfig struct {
-	URL           *config_util.URL `yaml:"url"`
+	URL           *config.URL    `yaml:"url"`
 	RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
 	ReadRecent    bool           `yaml:"read_recent,omitempty"`
 	Name          string         `yaml:"name,omitempty"`
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
-	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 
 	// RequiredMatchers is an optional list of equality matchers which have to
 	// be present in a selector to query the remote read endpoint.
 	RequiredMatchers model.LabelSet `yaml:"required_matchers,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *RemoteReadConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *RemoteReadConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultRemoteReadConfig

config/config_default_test.go

@@ -24,5 +24,4 @@ var ruleFilesExpectedConf = &Config{
 		"testdata/rules/second.rules",
 		"/absolute/third.rules",
 	},
-	original: "",
 }

config/config_test.go

@@ -23,12 +23,12 @@ import (
 	"testing"
 	"time"
 
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/azure"
-	sd_config "github.com/prometheus/prometheus/discovery/config"
 	"github.com/prometheus/prometheus/discovery/consul"
 	"github.com/prometheus/prometheus/discovery/digitalocean"
 	"github.com/prometheus/prometheus/discovery/dns"
@@ -46,12 +46,12 @@ import (
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
-func mustParseURL(u string) *config_util.URL {
+func mustParseURL(u string) *config.URL {
 	parsed, err := url.Parse(u)
 	if err != nil {
 		panic(err)
 	}
-	return &config_util.URL{URL: parsed}
+	return &config.URL{URL: parsed}
 }
 
 var expectedConf = &Config{
@@ -93,8 +93,8 @@ var expectedConf = &Config{
 			RemoteTimeout: model.Duration(30 * time.Second),
 			QueueConfig:   DefaultQueueConfig,
 			Name:          "rw_tls",
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
-				TLSConfig: config_util.TLSConfig{
+				TLSConfig: config.TLSConfig{
 					CertFile: filepath.FromSlash("testdata/valid_cert_file"),
 					KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 				},
@@ -115,8 +115,8 @@ var expectedConf = &Config{
 			ReadRecent:       false,
 			Name:             "read_special",
 			RequiredMatchers: model.LabelSet{"job": "special"},
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
-				TLSConfig: config_util.TLSConfig{
+				TLSConfig: config.TLSConfig{
 					CertFile: filepath.FromSlash("testdata/valid_cert_file"),
 					KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 				},
@@ -136,12 +136,20 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
 				BearerTokenFile: filepath.FromSlash("testdata/valid_token_file"),
 			},
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				StaticConfigs: []*targetgroup.Group{
+				&file.SDConfig{
+					Files:           []string{"testdata/foo/*.slow.json", "testdata/foo/*.slow.yml", "testdata/single/file.yml"},
+					RefreshInterval: model.Duration(10 * time.Minute),
+				},
+				&file.SDConfig{
+					Files:           []string{"testdata/bar/*.yaml"},
+					RefreshInterval: model.Duration(5 * time.Minute),
+				},
+				discovery.StaticConfig{
 					{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
@@ -154,17 +162,6 @@ var expectedConf = &Config{
 						Source: "0",
 					},
 				},
-
-				FileSDConfigs: []*file.SDConfig{
-					{
-						Files:           []string{"testdata/foo/*.slow.json", "testdata/foo/*.slow.yml", "testdata/single/file.yml"},
-						RefreshInterval: model.Duration(10 * time.Minute),
-					},
-					{
-						Files:           []string{"testdata/bar/*.yaml"},
-						RefreshInterval: model.Duration(5 * time.Minute),
-					},
-				},
 			},
 
 			RelabelConfigs: []*relabel.Config{
@@ -206,8 +203,8 @@ var expectedConf = &Config{
 			ScrapeTimeout:   model.Duration(5 * time.Second),
 			SampleLimit:     1000,
 
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
-				BasicAuth: &config_util.BasicAuth{
+				BasicAuth: &config.BasicAuth{
 					Username: "admin_name",
 					Password: "multiline\nmysecret\ntest",
 				},
@@ -215,9 +212,8 @@ var expectedConf = &Config{
 			MetricsPath: "/my_path",
 			Scheme:      "https",
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				DNSSDConfigs: []*dns.SDConfig{
+				&dns.SDConfig{
-					{
 					Names: []string{
 						"first.dns.address.domain.com",
 						"second.dns.address.domain.com",
@@ -225,7 +221,7 @@ var expectedConf = &Config{
 					RefreshInterval: model.Duration(15 * time.Second),
 					Type:            "SRV",
 				},
-					{
+				&dns.SDConfig{
 					Names: []string{
 						"first.dns.address.domain.com",
 					},
@@ -233,7 +229,6 @@ var expectedConf = &Config{
 					Type:            "SRV",
 				},
 			},
-			},
 
 			RelabelConfigs: []*relabel.Config{
 				{
@@ -298,9 +293,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				ConsulSDConfigs: []*consul.SDConfig{
+				&consul.SDConfig{
-					{
 					Server:          "localhost:1234",
 					Token:           "mysecret",
 					Services:        []string{"nginx", "cache", "mysql"},
@@ -310,7 +304,7 @@ var expectedConf = &Config{
 					Scheme:          "https",
 					RefreshInterval: consul.DefaultSDConfig.RefreshInterval,
 					AllowStale:      true,
-						TLSConfig: config_util.TLSConfig{
+					TLSConfig: config.TLSConfig{
 						CertFile:           filepath.FromSlash("testdata/valid_cert_file"),
 						KeyFile:            filepath.FromSlash("testdata/valid_key_file"),
 						CAFile:             filepath.FromSlash("testdata/valid_ca_file"),
@@ -318,7 +312,6 @@ var expectedConf = &Config{
 					},
 				},
 			},
-			},
 
 			RelabelConfigs: []*relabel.Config{
 				{
@@ -341,8 +334,8 @@ var expectedConf = &Config{
 			MetricsPath: "/metrics",
 			Scheme:      "http",
 
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
-				TLSConfig: config_util.TLSConfig{
+				TLSConfig: config.TLSConfig{
 					CertFile: filepath.FromSlash("testdata/valid_cert_file"),
 					KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 				},
@@ -360,17 +353,16 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				KubernetesSDConfigs: []*kubernetes.SDConfig{
+				&kubernetes.SDConfig{
-					{
 					APIServer: kubernetesSDHostURL(),
 					Role:      kubernetes.RoleEndpoint,
-						HTTPClientConfig: config_util.HTTPClientConfig{
+					HTTPClientConfig: config.HTTPClientConfig{
-							BasicAuth: &config_util.BasicAuth{
+						BasicAuth: &config.BasicAuth{
 							Username: "myusername",
 							Password: "mysecret",
 						},
-							TLSConfig: config_util.TLSConfig{
+						TLSConfig: config.TLSConfig{
 							CertFile: filepath.FromSlash("testdata/valid_cert_file"),
 							KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 						},
@@ -379,7 +371,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-kubernetes-namespaces",
 
@@ -389,16 +380,15 @@ var expectedConf = &Config{
 
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
-			HTTPClientConfig: config_util.HTTPClientConfig{
+			HTTPClientConfig: config.HTTPClientConfig{
-				BasicAuth: &config_util.BasicAuth{
+				BasicAuth: &config.BasicAuth{
 					Username:     "myusername",
 					PasswordFile: filepath.FromSlash("testdata/valid_password_file"),
 				},
 			},
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				KubernetesSDConfigs: []*kubernetes.SDConfig{
+				&kubernetes.SDConfig{
-					{
 					APIServer: kubernetesSDHostURL(),
 					Role:      kubernetes.RoleEndpoint,
 					NamespaceDiscovery: kubernetes.NamespaceDiscovery{
@@ -409,7 +399,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-marathon",
 
@@ -420,16 +409,15 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				MarathonSDConfigs: []*marathon.SDConfig{
+				&marathon.SDConfig{
-					{
 					Servers: []string{
 						"https://marathon.example.com:443",
 					},
 					RefreshInterval: model.Duration(30 * time.Second),
-						AuthToken:       config_util.Secret("mysecret"),
+					AuthToken:       "mysecret",
-						HTTPClientConfig: config_util.HTTPClientConfig{
+					HTTPClientConfig: config.HTTPClientConfig{
-							TLSConfig: config_util.TLSConfig{
+						TLSConfig: config.TLSConfig{
 							CertFile: filepath.FromSlash("testdata/valid_cert_file"),
 							KeyFile:  filepath.FromSlash("testdata/valid_key_file"),
 						},
@@ -437,7 +425,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-ec2",
 
@@ -448,9 +435,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				EC2SDConfigs: []*ec2.SDConfig{
+				&ec2.SDConfig{
-					{
 					Region:          "us-east-1",
 					AccessKey:       "access",
 					SecretKey:       "mysecret",
@@ -470,7 +456,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-azure",
 
@@ -481,9 +466,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				AzureSDConfigs: []*azure.SDConfig{
+				&azure.SDConfig{
-					{
 					Environment:          "AzurePublicCloud",
 					SubscriptionID:       "11AAAA11-A11A-111A-A111-1111A1111A11",
 					TenantID:             "BBBB222B-B2B2-2B22-B222-2BB2222BB2B2",
@@ -495,7 +479,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-nerve",
 
@@ -506,16 +489,14 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				NerveSDConfigs: []*zookeeper.NerveSDConfig{
+				&zookeeper.NerveSDConfig{
-					{
 					Servers: []string{"localhost"},
 					Paths:   []string{"/monitoring"},
 					Timeout: model.Duration(10 * time.Second),
 				},
 			},
 		},
-		},
 		{
 			JobName: "0123service-xxx",
 
@@ -526,8 +507,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				StaticConfigs: []*targetgroup.Group{
+				discovery.StaticConfig{
 					{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
@@ -547,8 +528,8 @@ var expectedConf = &Config{
 			MetricsPath: "/federate",
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				StaticConfigs: []*targetgroup.Group{
+				discovery.StaticConfig{
 					{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
@@ -568,8 +549,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				StaticConfigs: []*targetgroup.Group{
+				discovery.StaticConfig{
 					{
 						Targets: []model.LabelSet{
 							{model.AddressLabel: "localhost:9090"},
@@ -589,9 +570,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				TritonSDConfigs: []*triton.SDConfig{
+				&triton.SDConfig{
-					{
 					Account:         "testAccount",
 					Role:            "container",
 					DNSSuffix:       "triton.example.com",
@@ -599,14 +579,13 @@ var expectedConf = &Config{
 					Port:            9163,
 					RefreshInterval: model.Duration(60 * time.Second),
 					Version:         1,
-						TLSConfig: config_util.TLSConfig{
+					TLSConfig: config.TLSConfig{
 						CertFile: "testdata/valid_cert_file",
 						KeyFile:  "testdata/valid_key_file",
 					},
 				},
 			},
 		},
-		},
 		{
 			JobName: "digitalocean-droplets",
 
@@ -617,10 +596,9 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				DigitalOceanSDConfigs: []*digitalocean.SDConfig{
+				&digitalocean.SDConfig{
-					{
+					HTTPClientConfig: config.HTTPClientConfig{
-						HTTPClientConfig: config_util.HTTPClientConfig{
 						BearerToken: "abcdef",
 					},
 					Port:            80,
@@ -628,7 +606,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "dockerswarm",
 
@@ -639,9 +616,8 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{
-				DockerSwarmSDConfigs: []*dockerswarm.SDConfig{
+				&dockerswarm.SDConfig{
-					{
 					Host:            "http://127.0.0.1:2375",
 					Role:            "nodes",
 					Port:            80,
@@ -649,7 +625,6 @@ var expectedConf = &Config{
 				},
 			},
 		},
-		},
 		{
 			JobName: "service-openstack",
 
@@ -660,21 +635,17 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+			ServiceDiscoveryConfigs: discovery.Configs{&openstack.SDConfig{
-				OpenstackSDConfigs: []*openstack.SDConfig{
-					{
 				Role:            "instance",
 				Region:          "RegionOne",
 				Port:            80,
 				Availability:    "public",
 				RefreshInterval: model.Duration(60 * time.Second),
-						TLSConfig: config_util.TLSConfig{
+				TLSConfig: config.TLSConfig{
 					CAFile:   "testdata/valid_ca_file",
 					CertFile: "testdata/valid_cert_file",
 					KeyFile:  "testdata/valid_key_file",
-						},
+				}},
-					},
-				},
 			},
 		},
 	},
@@ -684,8 +655,8 @@ var expectedConf = &Config{
 				Scheme:     "https",
 				Timeout:    model.Duration(10 * time.Second),
 				APIVersion: AlertmanagerAPIVersionV1,
-				ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
+				ServiceDiscoveryConfigs: discovery.Configs{
-					StaticConfigs: []*targetgroup.Group{
+					discovery.StaticConfig{
 						{
 							Targets: []model.LabelSet{
 								{model.AddressLabel: "1.2.3.4:9093"},
@@ -699,7 +670,19 @@ var expectedConf = &Config{
 			},
 		},
 	},
-	original: "",
+}
+
+func TestYAMLRoundtrip(t *testing.T) {
+	want, err := LoadFile("testdata/roundtrip.good.yml")
+	testutil.Ok(t, err)
+
+	out, err := yaml.Marshal(want)
+
+	testutil.Ok(t, err)
+	got := &Config{}
+	testutil.Ok(t, yaml.UnmarshalStrict(out, got))
+
+	testutil.Equals(t, want, got)
 }
 
 func TestLoadConfig(t *testing.T) {
@@ -710,8 +693,6 @@ func TestLoadConfig(t *testing.T) {
 
 	c, err := LoadFile("testdata/conf.good.yml")
 	testutil.Ok(t, err)
-
-	expectedConf.original = c.original
 	testutil.Equals(t, expectedConf, c)
 }
 
@@ -745,8 +726,6 @@ func TestLoadConfigRuleFilesAbsolutePath(t *testing.T) {
 	// Parse a valid file that sets a rule files with an absolute path
 	c, err := LoadFile(ruleFilesConfigFile)
 	testutil.Ok(t, err)
-
-	ruleFilesExpectedConf.original = c.original
 	testutil.Equals(t, ruleFilesExpectedConf, c)
 }
 
@@ -834,7 +813,7 @@ var expectedErrors = []struct {
 		errMsg:   "invalid rule file path",
 	}, {
 		filename: "unknown_attr.bad.yml",
-		errMsg:   "field consult_sd_configs not found in type config.plain",
+		errMsg:   "field consult_sd_configs not found in type",
 	}, {
 		filename: "bearertoken.bad.yml",
 		errMsg:   "at most one of bearer_token & bearer_token_file must be configured",
@@ -1022,11 +1001,10 @@ func TestEmptyGlobalBlock(t *testing.T) {
 	c, err := Load("global:\n")
 	testutil.Ok(t, err)
 	exp := DefaultConfig
-	exp.original = "global:\n"
 	testutil.Equals(t, exp, *c)
 }
 
-func kubernetesSDHostURL() config_util.URL {
+func kubernetesSDHostURL() config.URL {
 	tURL, _ := url.Parse("https://localhost:1234")
-	return config_util.URL{URL: tURL}
+	return config.URL{URL: tURL}
 }

config/config_windows_test.go

@@ -22,5 +22,4 @@ var ruleFilesExpectedConf = &Config{
 		"testdata\\rules\\second.rules",
 		"c:\\absolute\\third.rules",
 	},
-	original: "",
 }

config/testdata/roundtrip.good.yml

@@ -0,0 +1,143 @@
+alerting:
+  alertmanagers:
+  - scheme: https
+
+    file_sd_configs:
+    - files:
+      - foo/*.slow.json
+      - foo/*.slow.yml
+      refresh_interval: 10m
+    - files:
+      - bar/*.yaml
+
+    static_configs:
+      - targets:
+        - 1.2.3.4:9093
+        - 1.2.3.5:9093
+        - 1.2.3.6:9093
+
+scrape_configs:
+
+- job_name: foo
+  static_configs:
+  - targets:
+    - localhost:9090
+    - localhost:9191
+    labels:
+      my:   label
+      your: label
+
+- job_name: bar
+
+  azure_sd_configs:
+  - environment: AzurePublicCloud
+    authentication_method: OAuth
+    subscription_id: 11AAAA11-A11A-111A-A111-1111A1111A11
+    tenant_id: BBBB222B-B2B2-2B22-B222-2BB2222BB2B2
+    client_id: 333333CC-3C33-3333-CCC3-33C3CCCCC33C
+    client_secret: <secret>
+    port: 9100
+
+  consul_sd_configs:
+  - server: localhost:1234
+    token: <secret>
+    services: [nginx, cache, mysql]
+    tags: [canary, v1]
+    node_meta:
+      rack: "123"
+    allow_stale: true
+    scheme: https
+    tls_config:
+      ca_file: valid_ca_file
+      cert_file: valid_cert_file
+      key_file:  valid_key_file
+
+  digitalocean_sd_configs:
+    - bearer_token: <secret>
+
+  dockerswarm_sd_configs:
+  - host: http://127.0.0.1:2375
+    role: nodes
+
+  dns_sd_configs:
+  - refresh_interval: 15s
+    names:
+    - first.dns.address.domain.com
+    - second.dns.address.domain.com
+  - names:
+    - first.dns.address.domain.com
+
+  ec2_sd_configs:
+  - region: us-east-1
+    access_key: access
+    secret_key: <secret>
+    profile: profile
+    filters:
+    - name: tag:environment
+      values:
+      - prod
+    - name: tag:service
+      values:
+      - web
+      - db
+
+  file_sd_configs:
+  - files:
+    - single/file.yml
+
+  kubernetes_sd_configs:
+  - role: endpoints
+    api_server: https://localhost:1234
+    tls_config:
+      cert_file: valid_cert_file
+      key_file: valid_key_file
+    basic_auth:
+      username: username
+      password: <secret>
+  - role: endpoints
+    api_server: https://localhost:1234
+    namespaces:
+      names:
+      - default
+    basic_auth:
+      username: username
+      password_file: valid_password_file
+
+  marathon_sd_configs:
+  - servers:
+    - https://marathon.example.com:443
+    auth_token: <secret>
+    tls_config:
+      cert_file: valid_cert_file
+      key_file: valid_key_file
+
+  nerve_sd_configs:
+  - servers:
+    - localhost
+    paths:
+    - /monitoring
+
+  openstack_sd_configs:
+  - role: instance
+    region: RegionOne
+    port: 80
+    refresh_interval: 1m
+    tls_config:
+      ca_file: valid_ca_file
+      cert_file: valid_cert_file
+      key_file:  valid_key_file
+
+  static_configs:
+  - targets:
+    - localhost:9093
+
+  triton_sd_configs:
+  - account: testAccount
+    dns_suffix: triton.example.com
+    endpoint: triton.example.com
+    port: 9163
+    refresh_interval: 1m
+    version: 1
+    tls_config:
+      cert_file: valid_cert_file
+      key_file: valid_key_file

discovery/README.md

@@ -146,7 +146,7 @@ both cases.
 
 For example if we had a discovery mechanism and it retrieves the following groups:
 
-```
+```go
 []targetgroup.Group{
 	{
 		Targets: []model.LabelSet{
@@ -161,7 +161,7 @@ For example if we had a discovery mechanism and it retrieves the following group
 				"test":         "simple-test",
 			},
 		},
-    Labels: map[LabelName][LabelValue] {
+		Labels: model.LabelSet{
 			"job": "mysql",
 		},
 		"Source": "file1",
@@ -179,7 +179,7 @@ For example if we had a discovery mechanism and it retrieves the following group
 				"test":         "simple-test",
 			},
 		},
-    Labels: map[LabelName][LabelValue] {
+		Labels: model.LabelSet{
 			"job": "postgres",
 		},
 		"Source": "file2",
@@ -190,7 +190,7 @@ For example if we had a discovery mechanism and it retrieves the following group
 Here there are two target groups one group with source `file1` and another with `file2`. The grouping is implementation specific and could even be one target per group. But, one has to make sure every target group sent by an SD instance should have a `Source` which is unique across all the target groups of that SD instance. 
 
 In this case, both the target groups are sent down the channel the first time `Run()` is called. Now, for an update, we need to send the whole _changed_ target group down the channel. i.e, if the target with `hostname: demo-postgres-2` goes away, we send:
-```
+```go
 &targetgroup.Group{
 	Targets: []model.LabelSet{
 		{
@@ -199,7 +199,7 @@ In this case, both the target groups are sent down the channel the first time `R
 			"test":         "simple-test",
 		},
 	},
-  Labels: map[LabelName][LabelValue] {
+	Labels: model.LabelSet{
 		"job": "postgres",
 	},
 	"Source": "file2",
@@ -208,7 +208,7 @@ In this case, both the target groups are sent down the channel the first time `R
 down the channel.
 
 If all the targets in a group go away, we need to send the target groups with empty `Targets` down the channel. i.e, if all targets with `job: postgres` go away, we send:
-```
+```go
 &targetgroup.Group{
 	Targets:  nil,
 	"Source": "file2",
@@ -216,15 +216,45 @@ If all the targets in a group go away, we need to send the target groups with em
 ```
 down the channel.
 
+### The Config interface
+
+Now that your service discovery mechanism is ready to discover targets, you must help
+Prometheus discover it. This is done by implementing the `discovery.Config` interface
+and registering it with `discovery.RegisterConfig` in an init function of your package.
+
+```go
+type Config interface {
+	// Name returns the name of the discovery mechanism.
+	Name() string
+
+	// NewDiscoverer returns a Discoverer for the Config
+	// with the given DiscovererOptions.
+	NewDiscoverer(DiscovererOptions) (Discoverer, error)
+}
+
+type DiscovererOptions struct {
+	Logger log.Logger
+}
+```
+
+The value returned by `Name()` should be short, descriptive, lowercase, and unique.
+It's used to tag the provided `Logger` and as the part of the YAML key for your SD
+mechanism's list of configs in `scrape_config` and `alertmanager_config`
+(e.g. `${NAME}_sd_configs`).
+
 ### New Service Discovery Check List
 
 Here are some non-obvious parts of adding service discoveries that need to be verified:
 
-- Check for `nil` SDConfigs in `discovery/config/config.go`.
 - Validate that discovery configs can be DeepEqualled by adding them to
   `config/testdata/conf.good.yml` and to the associated tests.
-- If there is a TLSConfig or HTTPClientConfig, add them to
+
-  `resolveFilepaths` in `config/config.go`.
+- If the config contains file paths directly or indirectly (e.g. with a TLSConfig or
+  HTTPClientConfig field), then it must implement `config.DirectorySetter`.
+
+- Import your SD package from `prometheus/discovery/install`. The install package is
+  imported from `main` to register all builtin SD mechanisms.
+
 - List the service discovery in both `<scrape_config>` and
   `<alertmanager_config>` in `docs/configuration/configuration.md`.
 

discovery/azure/azure.go

@@ -33,6 +33,7 @@ import (
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -64,6 +65,10 @@ var DefaultSDConfig = SDConfig{
 	AuthenticationMethod: authMethodOAuth,
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for Azure based service discovery.
 type SDConfig struct {
 	Environment          string             `yaml:"environment,omitempty"`
@@ -76,6 +81,14 @@ type SDConfig struct {
 	AuthenticationMethod string             `yaml:"authentication_method,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "azure" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger), nil
+}
+
 func validateAuthParam(param, name string) error {
 	if len(param) == 0 {
 		return errors.Errorf("azure SD configuration requires a %s", name)

discovery/config/config.go

@@ -1,147 +0,0 @@
-// Copyright 2016 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package config
-
-import (
-	"github.com/pkg/errors"
-
-	"github.com/prometheus/prometheus/discovery/azure"
-	"github.com/prometheus/prometheus/discovery/consul"
-	"github.com/prometheus/prometheus/discovery/digitalocean"
-	"github.com/prometheus/prometheus/discovery/dns"
-	"github.com/prometheus/prometheus/discovery/dockerswarm"
-	"github.com/prometheus/prometheus/discovery/ec2"
-	"github.com/prometheus/prometheus/discovery/file"
-	"github.com/prometheus/prometheus/discovery/gce"
-	"github.com/prometheus/prometheus/discovery/kubernetes"
-	"github.com/prometheus/prometheus/discovery/marathon"
-	"github.com/prometheus/prometheus/discovery/openstack"
-	"github.com/prometheus/prometheus/discovery/targetgroup"
-	"github.com/prometheus/prometheus/discovery/triton"
-	"github.com/prometheus/prometheus/discovery/zookeeper"
-)
-
-// ServiceDiscoveryConfig configures lists of different service discovery mechanisms.
-type ServiceDiscoveryConfig struct {
-	// List of labeled target groups for this job.
-	StaticConfigs []*targetgroup.Group `yaml:"static_configs,omitempty"`
-	// List of DNS service discovery configurations.
-	DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
-	// List of file service discovery configurations.
-	FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
-	// List of Consul service discovery configurations.
-	ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
-	// List of DigitalOcean service discovery configurations.
-	DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
-	// List of Docker Swarm service discovery configurations.
-	DockerSwarmSDConfigs []*dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
-	// List of Serverset service discovery configurations.
-	ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
-	// NerveSDConfigs is a list of Nerve service discovery configurations.
-	NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"`
-	// MarathonSDConfigs is a list of Marathon service discovery configurations.
-	MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"`
-	// List of Kubernetes service discovery configurations.
-	KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
-	// List of GCE service discovery configurations.
-	GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
-	// List of EC2 service discovery configurations.
-	EC2SDConfigs []*ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
-	// List of OpenStack service discovery configurations.
-	OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
-	// List of Azure service discovery configurations.
-	AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"`
-	// List of Triton service discovery configurations.
-	TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"`
-}
-
-// Validate validates the ServiceDiscoveryConfig.
-func (c *ServiceDiscoveryConfig) Validate() error {
-	for _, cfg := range c.AzureSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in azure_sd_configs")
-		}
-	}
-	for _, cfg := range c.ConsulSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in consul_sd_configs")
-		}
-	}
-	for _, cfg := range c.DigitalOceanSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in digitalocean_sd_configs")
-		}
-	}
-	for _, cfg := range c.DockerSwarmSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in dockerswarm_sd_configs")
-		}
-	}
-	for _, cfg := range c.DNSSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in dns_sd_configs")
-		}
-	}
-	for _, cfg := range c.EC2SDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in ec2_sd_configs")
-		}
-	}
-	for _, cfg := range c.FileSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in file_sd_configs")
-		}
-	}
-	for _, cfg := range c.GCESDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in gce_sd_configs")
-		}
-	}
-	for _, cfg := range c.KubernetesSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in kubernetes_sd_configs")
-		}
-	}
-	for _, cfg := range c.MarathonSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in marathon_sd_configs")
-		}
-	}
-	for _, cfg := range c.NerveSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in nerve_sd_configs")
-		}
-	}
-	for _, cfg := range c.OpenstackSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in openstack_sd_configs")
-		}
-	}
-	for _, cfg := range c.ServersetSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in serverset_sd_configs")
-		}
-	}
-	for _, cfg := range c.StaticConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in static_configs")
-		}
-	}
-	for _, cfg := range c.TritonSDConfigs {
-		if cfg == nil {
-			return errors.New("empty or null section in triton_sd_configs")
-		}
-	}
-	return nil
-}

discovery/config/config_test.go

@@ -1,58 +0,0 @@
-// Copyright 2020 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package config
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/prometheus/prometheus/util/testutil"
-	"gopkg.in/yaml.v2"
-)
-
-func TestForNilSDConfig(t *testing.T) {
-	// Get all the yaml fields names of the ServiceDiscoveryConfig struct.
-	s := reflect.ValueOf(ServiceDiscoveryConfig{})
-	configType := s.Type()
-	n := s.NumField()
-	fieldsSlice := make([]string, n)
-	for i := 0; i < n; i++ {
-		field := configType.Field(i)
-		tag := field.Tag.Get("yaml")
-		tag = strings.Split(tag, ",")[0]
-		fieldsSlice = append(fieldsSlice, tag)
-	}
-
-	// Unmarshall all possible yaml keys and validate errors check upon nil
-	// SD config.
-	for _, f := range fieldsSlice {
-		if f == "" {
-			continue
-		}
-		t.Run(f, func(t *testing.T) {
-			c := &ServiceDiscoveryConfig{}
-			err := yaml.Unmarshal([]byte(fmt.Sprintf(`
----
-%s:
--
-`, f)), c)
-			testutil.Ok(t, err)
-			err = c.Validate()
-			testutil.NotOk(t, err)
-			testutil.Equals(t, fmt.Sprintf("empty or null section in %s", f), err.Error())
-		})
-	}
-}

discovery/consul/consul.go

@@ -28,9 +28,10 @@ import (
 	conntrack "github.com/mwitkow/go-conntrack"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
 )
@@ -99,15 +100,21 @@ var (
 	}
 )
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+	prometheus.MustRegister(rpcFailuresCount)
+	prometheus.MustRegister(rpcDuration)
+}
+
 // SDConfig is the configuration for Consul service discovery.
 type SDConfig struct {
 	Server       string        `yaml:"server,omitempty"`
-	Token        config_util.Secret `yaml:"token,omitempty"`
+	Token        config.Secret `yaml:"token,omitempty"`
 	Datacenter   string        `yaml:"datacenter,omitempty"`
 	TagSeparator string        `yaml:"tag_separator,omitempty"`
 	Scheme       string        `yaml:"scheme,omitempty"`
 	Username     string        `yaml:"username,omitempty"`
-	Password     config_util.Secret `yaml:"password,omitempty"`
+	Password     config.Secret `yaml:"password,omitempty"`
 
 	// See https://www.consul.io/docs/internals/consensus.html#consistency-modes,
 	// stale reads are a lot cheaper and are a necessity if you have >5k targets.
@@ -127,7 +134,20 @@ type SDConfig struct {
 	// Desired node metadata.
 	NodeMeta map[string]string `yaml:"node_meta,omitempty"`
 
-	TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"`
+	TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"`
+}
+
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "consul" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.TLSConfig.SetDirectory(dir)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -144,11 +164,6 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-func init() {
-	prometheus.MustRegister(rpcFailuresCount)
-	prometheus.MustRegister(rpcDuration)
-}
-
 // Discovery retrieves target information from a Consul server
 // and updates them via watches.
 type Discovery struct {
@@ -170,7 +185,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		logger = log.NewNopLogger()
 	}
 
-	tls, err := config_util.NewTLSConfig(&conf.TLSConfig)
+	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
 	}

discovery/digitalocean/digitalocean.go

@@ -24,10 +24,11 @@ import (
 
 	"github.com/digitalocean/godo"
 	"github.com/go-kit/kit/log"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
@@ -54,14 +55,31 @@ var DefaultSDConfig = SDConfig{
 	RefreshInterval: model.Duration(60 * time.Second),
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for DigitalOcean based service discovery.
 type SDConfig struct {
-	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 	Port            int            `yaml:"port"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "digitalocean" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig
@@ -87,7 +105,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		port: conf.Port,
 	}
 
-	rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false)
+	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false, false)
 	if err != nil {
 		return nil, err
 	}

discovery/discovery.go

@@ -0,0 +1,117 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/go-kit/kit/log"
+
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+// Discoverer provides information about target groups. It maintains a set
+// of sources from which TargetGroups can originate. Whenever a discovery provider
+// detects a potential change, it sends the TargetGroup through its channel.
+//
+// Discoverer does not know if an actual change happened.
+// It does guarantee that it sends the new TargetGroup whenever a change happens.
+//
+// Discoverers should initially send a full set of all discoverable TargetGroups.
+type Discoverer interface {
+	// Run hands a channel to the discovery provider (Consul, DNS, etc.) through which
+	// it can send updated target groups. It must return when the context is canceled.
+	// It should not close the update channel on returning.
+	Run(ctx context.Context, up chan<- []*targetgroup.Group)
+}
+
+// DiscovererOptions provides options for a Discoverer.
+type DiscovererOptions struct {
+	Logger log.Logger
+}
+
+// A Config provides the configuration and constructor for a Discoverer.
+type Config interface {
+	// Name returns the name of the discovery mechanism.
+	Name() string
+
+	// NewDiscoverer returns a Discoverer for the Config
+	// with the given DiscovererOptions.
+	NewDiscoverer(DiscovererOptions) (Discoverer, error)
+}
+
+// Configs is a slice of Config values that uses custom YAML marshaling and unmarshaling
+// to represent itself as a mapping of the Config values grouped by their types.
+type Configs []Config
+
+// SetDirectory joins any relative file paths with dir.
+func (c *Configs) SetDirectory(dir string) {
+	for _, c := range *c {
+		if v, ok := c.(config.DirectorySetter); ok {
+			v.SetDirectory(dir)
+		}
+	}
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (c *Configs) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	cfgTyp := getConfigType(configsType)
+	cfgPtr := reflect.New(cfgTyp)
+	cfgVal := cfgPtr.Elem()
+
+	if err := unmarshal(cfgPtr.Interface()); err != nil {
+		return replaceYAMLTypeError(err, cfgTyp, configsType)
+	}
+
+	var err error
+	*c, err = readConfigs(cfgVal, 0)
+	return err
+}
+
+// MarshalYAML implements yaml.Marshaler.
+func (c Configs) MarshalYAML() (interface{}, error) {
+	cfgTyp := getConfigType(configsType)
+	cfgPtr := reflect.New(cfgTyp)
+	cfgVal := cfgPtr.Elem()
+
+	if err := writeConfigs(cfgVal, c); err != nil {
+		return nil, err
+	}
+
+	return cfgPtr.Interface(), nil
+}
+
+// A StaticConfig is a Config that provides a static list of targets.
+type StaticConfig []*targetgroup.Group
+
+// Name returns the name of the service discovery mechanism.
+func (StaticConfig) Name() string { return "static" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
+	return staticDiscoverer(c), nil
+}
+
+type staticDiscoverer []*targetgroup.Group
+
+func (c staticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {
+	// TODO: existing implementation closes up chan, but documentation explicitly forbids it...?
+	defer close(up)
+	select {
+	case <-ctx.Done():
+	case up <- c:
+	}
+}

discovery/dns/dns.go

@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
@@ -65,6 +66,12 @@ var (
 	}
 )
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+	prometheus.MustRegister(dnsSDLookupFailuresCount)
+	prometheus.MustRegister(dnsSDLookupsCount)
+}
+
 // SDConfig is the configuration for DNS based service discovery.
 type SDConfig struct {
 	Names           []string       `yaml:"names"`
@@ -73,6 +80,14 @@ type SDConfig struct {
 	Port            int            `yaml:"port"` // Ignored for SRV records
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "dns" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(*c, opts.Logger), nil
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig
@@ -96,11 +111,6 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-func init() {
-	prometheus.MustRegister(dnsSDLookupFailuresCount)
-	prometheus.MustRegister(dnsSDLookupsCount)
-}
-
 // Discovery periodically performs DNS-SD requests. It implements
 // the Discoverer interface.
 type Discovery struct {

discovery/dockerswarm/dockerswarm.go

@@ -22,10 +22,11 @@ import (
 
 	"github.com/docker/docker/client"
 	"github.com/go-kit/kit/log"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
@@ -42,9 +43,13 @@ var DefaultSDConfig = SDConfig{
 	Port:            80,
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for Docker Swarm based service discovery.
 type SDConfig struct {
-	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 
 	Host string `yaml:"host"`
 	Role string `yaml:"role"`
@@ -53,6 +58,19 @@ type SDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "dockerswarm" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig
@@ -109,7 +127,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	// unix, which are not supported by the HTTP client. Passing HTTP client
 	// options to the Docker client makes those non-HTTP requests fail.
 	if hostURL.Scheme == "http" || hostURL.Scheme == "https" {
-		rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "dockerswarm_sd", false)
+		rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "dockerswarm_sd", false, false)
 		if err != nil {
 			return nil, err
 		}

discovery/ec2/ec2.go

@@ -28,9 +28,10 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -64,6 +65,10 @@ var DefaultSDConfig = SDConfig{
 	RefreshInterval: model.Duration(60 * time.Second),
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // Filter is the configuration for filtering EC2 instances.
 type Filter struct {
 	Name   string   `yaml:"name"`
@@ -75,7 +80,7 @@ type SDConfig struct {
 	Endpoint        string         `yaml:"endpoint"`
 	Region          string         `yaml:"region"`
 	AccessKey       string         `yaml:"access_key,omitempty"`
-	SecretKey       config_util.Secret `yaml:"secret_key,omitempty"`
+	SecretKey       config.Secret  `yaml:"secret_key,omitempty"`
 	Profile         string         `yaml:"profile,omitempty"`
 	RoleARN         string         `yaml:"role_arn,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
@@ -83,6 +88,14 @@ type SDConfig struct {
 	Filters         []*Filter      `yaml:"filters"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "ec2" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger), nil
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig

discovery/file/file.go

@@ -29,10 +29,12 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	fsnotify "gopkg.in/fsnotify/fsnotify.v1"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -45,12 +47,31 @@ var (
 	}
 )
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for file based discovery.
 type SDConfig struct {
 	Files           []string       `yaml:"files"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "file" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger), nil
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	for i, file := range c.Files {
+		c.Files[i] = config.JoinDir(dir, file)
+	}
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig

discovery/gce/gce.go

@@ -28,6 +28,7 @@ import (
 	compute "google.golang.org/api/compute/v1"
 	"google.golang.org/api/option"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -57,6 +58,10 @@ var DefaultSDConfig = SDConfig{
 	RefreshInterval: model.Duration(60 * time.Second),
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for GCE based service discovery.
 type SDConfig struct {
 	// Project: The Google Cloud Project ID
@@ -76,6 +81,14 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "gce" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(*c, opts.Logger)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig

discovery/install/install.go

@@ -0,0 +1,32 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package install has the side-effect of registering all builtin
+// service discovery config types.
+package install
+
+import (
+	_ "github.com/prometheus/prometheus/discovery/azure"        // register azure
+	_ "github.com/prometheus/prometheus/discovery/consul"       // register consul
+	_ "github.com/prometheus/prometheus/discovery/digitalocean" // register digitalocean
+	_ "github.com/prometheus/prometheus/discovery/dns"          // register dns
+	_ "github.com/prometheus/prometheus/discovery/dockerswarm"  // register dockerswarm
+	_ "github.com/prometheus/prometheus/discovery/ec2"          // register ec2
+	_ "github.com/prometheus/prometheus/discovery/file"         // register file
+	_ "github.com/prometheus/prometheus/discovery/gce"          // register gce
+	_ "github.com/prometheus/prometheus/discovery/kubernetes"   // register kubernetes
+	_ "github.com/prometheus/prometheus/discovery/marathon"     // register marathon
+	_ "github.com/prometheus/prometheus/discovery/openstack"    // register openstack
+	_ "github.com/prometheus/prometheus/discovery/triton"       // register triton
+	_ "github.com/prometheus/prometheus/discovery/zookeeper"    // register zookeeper
+)

discovery/kubernetes/kubernetes.go

@@ -25,8 +25,9 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/version"
 	apiv1 "k8s.io/api/core/v1"
 	disv1beta1 "k8s.io/api/discovery/v1beta1"
 	"k8s.io/api/networking/v1beta1"
@@ -38,7 +39,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 
-	"github.com/prometheus/common/version"
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -67,6 +68,19 @@ var (
 	DefaultSDConfig = SDConfig{}
 )
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+	prometheus.MustRegister(eventCount)
+	// Initialize metric vectors.
+	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
+		for _, evt := range []string{"add", "delete", "update"} {
+			eventCount.WithLabelValues(role, evt)
+		}
+	}
+	(&clientGoRequestMetricAdapter{}).Register(prometheus.DefaultRegisterer)
+	(&clientGoWorkqueueMetricsProvider{}).Register(prometheus.DefaultRegisterer)
+}
+
 // Role is role of the service in Kubernetes.
 type Role string
 
@@ -95,13 +109,26 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 // SDConfig is the configuration for Kubernetes service discovery.
 type SDConfig struct {
-	APIServer          config_util.URL              `yaml:"api_server,omitempty"`
+	APIServer          config.URL              `yaml:"api_server,omitempty"`
 	Role               Role                    `yaml:"role"`
-	HTTPClientConfig   config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig   config.HTTPClientConfig `yaml:",inline"`
 	NamespaceDiscovery NamespaceDiscovery      `yaml:"namespaces,omitempty"`
 	Selectors          []SelectorConfig        `yaml:"selectors,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "kubernetes" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return New(opts.Logger, c)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
 type roleSelector struct {
 	node          resourceSelector
 	pod           resourceSelector
@@ -137,7 +164,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err != nil {
 		return err
 	}
-	if c.APIServer.URL == nil && !reflect.DeepEqual(c.HTTPClientConfig, config_util.HTTPClientConfig{}) {
+	if c.APIServer.URL == nil && !reflect.DeepEqual(c.HTTPClientConfig, config.HTTPClientConfig{}) {
 		return errors.Errorf("to use custom HTTP client configuration please provide the 'api_server' URL explicitly")
 	}
 
@@ -197,31 +224,6 @@ func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) er
 	return unmarshal((*plain)(c))
 }
 
-func init() {
-	prometheus.MustRegister(eventCount)
-
-	// Initialize metric vectors.
-	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
-		for _, evt := range []string{"add", "delete", "update"} {
-			eventCount.WithLabelValues(role, evt)
-		}
-	}
-
-	var (
-		clientGoRequestMetricAdapterInstance     = clientGoRequestMetricAdapter{}
-		clientGoWorkqueueMetricsProviderInstance = clientGoWorkqueueMetricsProvider{}
-	)
-
-	clientGoRequestMetricAdapterInstance.Register(prometheus.DefaultRegisterer)
-	clientGoWorkqueueMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
-
-}
-
-// This is only for internal use.
-type discoverer interface {
-	Run(ctx context.Context, up chan<- []*targetgroup.Group)
-}
-
 // Discovery implements the discoverer interface for discovering
 // targets from Kubernetes.
 type Discovery struct {
@@ -230,7 +232,7 @@ type Discovery struct {
 	role               Role
 	logger             log.Logger
 	namespaceDiscovery *NamespaceDiscovery
-	discoverers        []discoverer
+	discoverers        []discovery.Discoverer
 	selectors          roleSelector
 }
 
@@ -260,7 +262,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		}
 		level.Info(l).Log("msg", "Using pod service account via in-cluster config")
 	} else {
-		rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "kubernetes_sd", false)
+		rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "kubernetes_sd", false, false)
 		if err != nil {
 			return nil, err
 		}
@@ -281,7 +283,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		logger:             l,
 		role:               conf.Role,
 		namespaceDiscovery: &conf.NamespaceDiscovery,
-		discoverers:        make([]discoverer, 0),
+		discoverers:        make([]discovery.Discoverer, 0),
 		selectors:          mapSelector(conf.Selectors),
 	}, nil
 }
@@ -517,7 +519,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	var wg sync.WaitGroup
 	for _, dd := range d.discoverers {
 		wg.Add(1)
-		go func(d discoverer) {
+		go func(d discovery.Discoverer) {
 			defer wg.Done()
 			d.Run(ctx, ch)
 		}(dd)

discovery/kubernetes/kubernetes_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/testutil"
 )
@@ -47,7 +48,7 @@ func makeDiscovery(role Role, nsDiscovery NamespaceDiscovery, objects ...runtime
 
 type k8sDiscoveryTest struct {
 	// discovery is instance of discovery.Discoverer
-	discovery discoverer
+	discovery discovery.Discoverer
 	// beforeRun runs before discoverer run
 	beforeRun func()
 	// afterStart runs after discoverer has synced

discovery/manager.go

@@ -24,22 +24,7 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 
-	sd_config "github.com/prometheus/prometheus/discovery/config"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
-
-	"github.com/prometheus/prometheus/discovery/azure"
-	"github.com/prometheus/prometheus/discovery/consul"
-	"github.com/prometheus/prometheus/discovery/digitalocean"
-	"github.com/prometheus/prometheus/discovery/dns"
-	"github.com/prometheus/prometheus/discovery/dockerswarm"
-	"github.com/prometheus/prometheus/discovery/ec2"
-	"github.com/prometheus/prometheus/discovery/file"
-	"github.com/prometheus/prometheus/discovery/gce"
-	"github.com/prometheus/prometheus/discovery/kubernetes"
-	"github.com/prometheus/prometheus/discovery/marathon"
-	"github.com/prometheus/prometheus/discovery/openstack"
-	"github.com/prometheus/prometheus/discovery/triton"
-	"github.com/prometheus/prometheus/discovery/zookeeper"
 )
 
 var (
@@ -84,22 +69,6 @@ func init() {
 	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
 }
 
-// Discoverer provides information about target groups. It maintains a set
-// of sources from which TargetGroups can originate. Whenever a discovery provider
-// detects a potential change, it sends the TargetGroup through its channel.
-//
-// Discoverer does not know if an actual change happened.
-// It does guarantee that it sends the new TargetGroup whenever a change happens.
-//
-// Discoverers should initially send a full set of all discoverable TargetGroups.
-type Discoverer interface {
-	// Run hands a channel to the discovery provider (Consul, DNS etc) through which it can send
-	// updated target groups.
-	// Must returns if the context gets canceled. It should not close the update
-	// channel on returning.
-	Run(ctx context.Context, up chan<- []*targetgroup.Group)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -183,7 +152,7 @@ func (m *Manager) SyncCh() <-chan map[string][]*targetgroup.Group {
 }
 
 // ApplyConfig removes all running discovery providers and starts new ones using the provided config.
-func (m *Manager) ApplyConfig(cfg map[string]sd_config.ServiceDiscoveryConfig) error {
+func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 
@@ -324,13 +293,12 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 }
 
 // registerProviders returns a number of failed SD config.
-func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setName string) int {
+func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 	var (
-		failedCount int
+		failed int
 		added  bool
 	)
-	add := func(cfg interface{}, newDiscoverer func() (Discoverer, error)) {
+	add := func(cfg Config) {
-		t := reflect.TypeOf(cfg).String()
 		for _, p := range m.providers {
 			if reflect.DeepEqual(cfg, p.config) {
 				p.subs = append(p.subs, setName)
@@ -338,98 +306,25 @@ func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setNam
 				return
 			}
 		}
-
+		typ := cfg.Name()
-		d, err := newDiscoverer()
+		d, err := cfg.NewDiscoverer(DiscovererOptions{
+			Logger: log.With(m.logger, "discovery", typ),
+		})
 		if err != nil {
-			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", t)
+			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ)
-			failedCount++
+			failed++
 			return
 		}
-
+		m.providers = append(m.providers, &provider{
-		provider := provider{
+			name:   fmt.Sprintf("%s/%d", typ, len(m.providers)),
-			name:   fmt.Sprintf("%s/%d", t, len(m.providers)),
 			d:      d,
 			config: cfg,
 			subs:   []string{setName},
-		}
+		})
-		m.providers = append(m.providers, &provider)
 		added = true
 	}
-
+	for _, cfg := range cfgs {
-	for _, c := range cfg.DNSSDConfigs {
+		add(cfg)
-		add(c, func() (Discoverer, error) {
-			return dns.NewDiscovery(*c, log.With(m.logger, "discovery", "dns")), nil
-		})
-	}
-	for _, c := range cfg.FileSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return file.NewDiscovery(c, log.With(m.logger, "discovery", "file")), nil
-		})
-	}
-	for _, c := range cfg.ConsulSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return consul.NewDiscovery(c, log.With(m.logger, "discovery", "consul"))
-		})
-	}
-	for _, c := range cfg.DigitalOceanSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return digitalocean.NewDiscovery(c, log.With(m.logger, "discovery", "digitalocean"))
-		})
-	}
-	for _, c := range cfg.DockerSwarmSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return dockerswarm.NewDiscovery(c, log.With(m.logger, "discovery", "dockerswarm"))
-		})
-	}
-	for _, c := range cfg.MarathonSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return marathon.NewDiscovery(*c, log.With(m.logger, "discovery", "marathon"))
-		})
-	}
-	for _, c := range cfg.KubernetesSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return kubernetes.New(log.With(m.logger, "discovery", "k8s"), c)
-		})
-	}
-	for _, c := range cfg.ServersetSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return zookeeper.NewServersetDiscovery(c, log.With(m.logger, "discovery", "zookeeper"))
-		})
-	}
-	for _, c := range cfg.NerveSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return zookeeper.NewNerveDiscovery(c, log.With(m.logger, "discovery", "nerve"))
-		})
-	}
-	for _, c := range cfg.EC2SDConfigs {
-		add(c, func() (Discoverer, error) {
-			return ec2.NewDiscovery(c, log.With(m.logger, "discovery", "ec2")), nil
-		})
-	}
-	for _, c := range cfg.OpenstackSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return openstack.NewDiscovery(c, log.With(m.logger, "discovery", "openstack"))
-		})
-	}
-	for _, c := range cfg.GCESDConfigs {
-		add(c, func() (Discoverer, error) {
-			return gce.NewDiscovery(*c, log.With(m.logger, "discovery", "gce"))
-		})
-	}
-	for _, c := range cfg.AzureSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return azure.NewDiscovery(c, log.With(m.logger, "discovery", "azure")), nil
-		})
-	}
-	for _, c := range cfg.TritonSDConfigs {
-		add(c, func() (Discoverer, error) {
-			return triton.New(log.With(m.logger, "discovery", "triton"), c)
-		})
-	}
-	if len(cfg.StaticConfigs) > 0 {
-		add(setName, func() (Discoverer, error) {
-			return &StaticProvider{TargetGroups: cfg.StaticConfigs}, nil
-		})
 	}
 	if !added {
 		// Add an empty target group to force the refresh of the corresponding
@@ -437,11 +332,9 @@ func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setNam
 		// current targets.
 		// It can happen because the combined set of SD configurations is empty
 		// or because we fail to instantiate all the SD configurations.
-		add(setName, func() (Discoverer, error) {
+		add(StaticConfig{{}})
-			return &StaticProvider{TargetGroups: []*targetgroup.Group{{}}}, nil
-		})
 	}
-	return failedCount
+	return failed
 }
 
 // StaticProvider holds a list of target groups that never change.

discovery/manager_test.go

@@ -16,8 +16,6 @@ package discovery
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"reflect"
 	"sort"
 	"strconv"
@@ -26,15 +24,9 @@ import (
 
 	"github.com/go-kit/kit/log"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
-	common_config "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
-	"github.com/prometheus/prometheus/config"
-	sd_config "github.com/prometheus/prometheus/discovery/config"
-	"github.com/prometheus/prometheus/discovery/consul"
-	"github.com/prometheus/prometheus/discovery/file"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/testutil"
-	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
@@ -725,6 +717,19 @@ func assertEqualGroups(t *testing.T, got, expected []*targetgroup.Group, msg fun
 
 }
 
+func staticConfig(addrs ...string) StaticConfig {
+	var cfg StaticConfig
+	for i, addr := range addrs {
+		cfg = append(cfg, &targetgroup.Group{
+			Source: fmt.Sprint(i),
+			Targets: []model.LabelSet{
+				{model.AddressLabel: model.LabelValue(addr)},
+			},
+		})
+	}
+	return cfg
+}
+
 func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
 	t.Helper()
 	if _, ok := tSets[poolKey]; !ok {
@@ -760,51 +765,46 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
-	c := map[string]sd_config.ServiceDiscoveryConfig{
+	c := map[string]Configs{
 		"prometheus": {
-			StaticConfigs: []*targetgroup.Group{
+			staticConfig("foo:9090", "bar:9090"),
-				{
-					Source: "0",
-					Targets: []model.LabelSet{
-						{
-							model.AddressLabel: model.LabelValue("foo:9090"),
-						},
-					},
-				},
-				{
-					Source: "1",
-					Targets: []model.LabelSet{
-						{
-							model.AddressLabel: model.LabelValue("bar:9090"),
-						},
-					},
-				},
-			},
 		},
 	}
 	discoveryManager.ApplyConfig(c)
 
 	<-discoveryManager.SyncCh()
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "string/0"}, "{__address__=\"foo:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "string/0"}, "{__address__=\"bar:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
 
-	c["prometheus"] = sd_config.ServiceDiscoveryConfig{
+	c["prometheus"] = Configs{
-		StaticConfigs: []*targetgroup.Group{
+		staticConfig("foo:9090"),
-			{
+	}
-				Source: "0",
+	discoveryManager.ApplyConfig(c)
-				Targets: []model.LabelSet{
+
-					{
+	<-discoveryManager.SyncCh()
-						model.AddressLabel: model.LabelValue("foo:9090"),
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
-					},
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", false)
-				},
+}
-			},
+
+func TestDiscovererConfigs(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager.updatert = 100 * time.Millisecond
+	go discoveryManager.Run()
+
+	c := map[string]Configs{
+		"prometheus": {
+			staticConfig("foo:9090", "bar:9090"),
+			staticConfig("baz:9090"),
 		},
 	}
 	discoveryManager.ApplyConfig(c)
 
 	<-discoveryManager.SyncCh()
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "string/0"}, "{__address__=\"foo:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "string/0"}, "{__address__=\"bar:9090\"}", false)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/1"}, "{__address__=\"baz:9090\"}", true)
 }
 
 // TestTargetSetRecreatesEmptyStaticConfigs ensures that reloading a config file after
@@ -817,33 +817,24 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
-	c := map[string]sd_config.ServiceDiscoveryConfig{
+	c := map[string]Configs{
 		"prometheus": {
-			StaticConfigs: []*targetgroup.Group{
+			staticConfig("foo:9090"),
-				{
-					Source: "0",
-					Targets: []model.LabelSet{
-						{
-							model.AddressLabel: model.LabelValue("foo:9090"),
-						},
-					},
-				},
-			},
 		},
 	}
 	discoveryManager.ApplyConfig(c)
 
 	<-discoveryManager.SyncCh()
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "string/0"}, "{__address__=\"foo:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
 
-	c["prometheus"] = sd_config.ServiceDiscoveryConfig{
+	c["prometheus"] = Configs{
-		StaticConfigs: []*targetgroup.Group{},
+		StaticConfig{{}},
 	}
 	discoveryManager.ApplyConfig(c)
 
 	<-discoveryManager.SyncCh()
 
-	pkey := poolKey{setName: "prometheus", provider: "string/0"}
+	pkey := poolKey{setName: "prometheus", provider: "static/0"}
 	targetGroups, ok := discoveryManager.targets[pkey]
 	if !ok {
 		t.Fatalf("'%v' should be present in target groups", pkey)
@@ -859,78 +850,36 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 }
 
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
-	tmpFile, err := ioutil.TempFile("", "sd")
-	if err != nil {
-		t.Fatalf("error creating temporary file: %v", err)
-	}
-	defer os.Remove(tmpFile.Name())
-	if _, err := tmpFile.Write([]byte(`[{"targets": ["foo:9090"]}]`)); err != nil {
-		t.Fatalf("error writing temporary file: %v", err)
-	}
-	if err := tmpFile.Close(); err != nil {
-		t.Fatalf("error closing temporary file: %v", err)
-	}
-	tmpFile2 := fmt.Sprintf("%s.json", tmpFile.Name())
-	if err = os.Link(tmpFile.Name(), tmpFile2); err != nil {
-		t.Fatalf("error linking temporary file: %v", err)
-	}
-	defer os.Remove(tmpFile2)
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	discoveryManager := NewManager(ctx, nil)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
-	c := map[string]sd_config.ServiceDiscoveryConfig{
+	c := map[string]Configs{
 		"prometheus": {
-			FileSDConfigs: []*file.SDConfig{
+			staticConfig("foo:9090"),
-				{
-					Files: []string{
-						tmpFile2,
-					},
-					RefreshInterval: file.DefaultSDConfig.RefreshInterval,
-				},
-			},
 		},
 		"prometheus2": {
-			FileSDConfigs: []*file.SDConfig{
+			staticConfig("foo:9090"),
-				{
-					Files: []string{
-						tmpFile2,
-					},
-					RefreshInterval: file.DefaultSDConfig.RefreshInterval,
-				},
-			},
 		},
 	}
 	discoveryManager.ApplyConfig(c)
 
 	<-discoveryManager.SyncCh()
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "*file.SDConfig/0"}, "{__address__=\"foo:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
-	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus2", provider: "*file.SDConfig/0"}, "{__address__=\"foo:9090\"}", true)
+	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus2", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
 	if len(discoveryManager.providers) != 1 {
 		t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
 	}
 }
 
-func TestApplyConfigDoesNotModifyStaticProviderTargets(t *testing.T) {
+func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
-	cfgText := `
+	originalConfig := Configs{
-scrape_configs:
+		staticConfig("foo:9090", "bar:9090", "baz:9090"),
- - job_name: 'prometheus'
-   static_configs:
-   - targets: ["foo:9090"]
-   - targets: ["bar:9090"]
-   - targets: ["baz:9090"]
-`
-	originalConfig := &config.Config{}
-	if err := yaml.UnmarshalStrict([]byte(cfgText), originalConfig); err != nil {
-		t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
 	}
-
+	processedConfig := Configs{
-	processedConfig := &config.Config{}
+		staticConfig("foo:9090", "bar:9090", "baz:9090"),
-	if err := yaml.UnmarshalStrict([]byte(cfgText), processedConfig); err != nil {
-		t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -938,21 +887,25 @@ scrape_configs:
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
-	c := map[string]sd_config.ServiceDiscoveryConfig{
+	cfgs := map[string]Configs{
-		"prometheus": processedConfig.ScrapeConfigs[0].ServiceDiscoveryConfig,
+		"prometheus": processedConfig,
 	}
-	discoveryManager.ApplyConfig(c)
+	discoveryManager.ApplyConfig(cfgs)
 	<-discoveryManager.SyncCh()
 
-	origSdcfg := originalConfig.ScrapeConfigs[0].ServiceDiscoveryConfig
+	for _, cfg := range cfgs {
-	for _, sdcfg := range c {
+		if !reflect.DeepEqual(originalConfig, cfg) {
-		if !reflect.DeepEqual(origSdcfg.StaticConfigs, sdcfg.StaticConfigs) {
 			t.Fatalf("discovery manager modified static config \n  expected: %v\n  got: %v\n",
-				origSdcfg.StaticConfigs, sdcfg.StaticConfigs)
+				originalConfig, cfg)
 		}
 	}
 }
 
+type errorConfig struct{ err error }
+
+func (e errorConfig) Name() string                                        { return "error" }
+func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }
+
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -960,28 +913,11 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
-	c := map[string]sd_config.ServiceDiscoveryConfig{
+	c := map[string]Configs{
 		"prometheus": {
-			ConsulSDConfigs: []*consul.SDConfig{
+			errorConfig{fmt.Errorf("tests error 0")},
-				{
+			errorConfig{fmt.Errorf("tests error 1")},
-					Server: "foo:8500",
+			errorConfig{fmt.Errorf("tests error 2")},
-					TLSConfig: common_config.TLSConfig{
-						CertFile: "/tmp/non_existent",
-					},
-				},
-				{
-					Server: "bar:8500",
-					TLSConfig: common_config.TLSConfig{
-						CertFile: "/tmp/non_existent",
-					},
-				},
-				{
-					Server: "foo2:8500",
-					TLSConfig: common_config.TLSConfig{
-						CertFile: "/tmp/non_existent",
-					},
-				},
-			},
 		},
 	}
 	discoveryManager.ApplyConfig(c)
@@ -992,17 +928,8 @@ func TestGaugeFailedConfigs(t *testing.T) {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
 
-	c["prometheus"] = sd_config.ServiceDiscoveryConfig{
+	c["prometheus"] = Configs{
-		StaticConfigs: []*targetgroup.Group{
+		staticConfig("foo:9090"),
-			{
-				Source: "0",
-				Targets: []model.LabelSet{
-					{
-						model.AddressLabel: "foo:9090",
-					},
-				},
-			},
-		},
 	}
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()

discovery/marathon/marathon.go

@@ -28,9 +28,10 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -63,13 +64,31 @@ var DefaultSDConfig = SDConfig{
 	RefreshInterval: model.Duration(30 * time.Second),
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for services running on Marathon.
 type SDConfig struct {
 	Servers          []string                `yaml:"servers,omitempty"`
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
-	AuthToken        config_util.Secret           `yaml:"auth_token,omitempty"`
+	AuthToken        config.Secret           `yaml:"auth_token,omitempty"`
 	AuthTokenFile    string                  `yaml:"auth_token_file,omitempty"`
-	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
+}
+
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "marathon" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(*c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+	c.AuthTokenFile = config.JoinDir(dir, c.AuthTokenFile)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -108,7 +127,7 @@ type Discovery struct {
 
 // NewDiscovery returns a new Marathon Discovery.
 func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
-	rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd", false)
+	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd", false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -137,12 +156,12 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 }
 
 type authTokenRoundTripper struct {
-	authToken config_util.Secret
+	authToken config.Secret
 	rt        http.RoundTripper
 }
 
 // newAuthTokenRoundTripper adds the provided auth token to a request.
-func newAuthTokenRoundTripper(token config_util.Secret, rt http.RoundTripper) (http.RoundTripper, error) {
+func newAuthTokenRoundTripper(token config.Secret, rt http.RoundTripper) (http.RoundTripper, error) {
 	return &authTokenRoundTripper{token, rt}, nil
 }
 

discovery/openstack/openstack.go

@@ -24,9 +24,10 @@ import (
 	"github.com/gophercloud/gophercloud/openstack"
 	conntrack "github.com/mwitkow/go-conntrack"
 	"github.com/pkg/errors"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
@@ -38,28 +39,45 @@ var DefaultSDConfig = SDConfig{
 	Availability:    "public",
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for OpenStack based service discovery.
 type SDConfig struct {
 	IdentityEndpoint            string           `yaml:"identity_endpoint"`
 	Username                    string           `yaml:"username"`
 	UserID                      string           `yaml:"userid"`
-	Password                    config_util.Secret    `yaml:"password"`
+	Password                    config.Secret    `yaml:"password"`
 	ProjectName                 string           `yaml:"project_name"`
 	ProjectID                   string           `yaml:"project_id"`
 	DomainName                  string           `yaml:"domain_name"`
 	DomainID                    string           `yaml:"domain_id"`
 	ApplicationCredentialName   string           `yaml:"application_credential_name"`
 	ApplicationCredentialID     string           `yaml:"application_credential_id"`
-	ApplicationCredentialSecret config_util.Secret    `yaml:"application_credential_secret"`
+	ApplicationCredentialSecret config.Secret    `yaml:"application_credential_secret"`
 	Role                        Role             `yaml:"role"`
 	Region                      string           `yaml:"region"`
 	RefreshInterval             model.Duration   `yaml:"refresh_interval"`
 	Port                        int              `yaml:"port"`
 	AllTenants                  bool             `yaml:"all_tenants,omitempty"`
-	TLSConfig                   config_util.TLSConfig `yaml:"tls_config,omitempty"`
+	TLSConfig                   config.TLSConfig `yaml:"tls_config,omitempty"`
 	Availability                string           `yaml:"availability,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "openstack" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.TLSConfig.SetDirectory(dir)
+}
+
 // Role is the role of the target in OpenStack.
 type Role string
 
@@ -157,7 +175,7 @@ func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) {
 	if err != nil {
 		return nil, err
 	}
-	tls, err := config_util.NewTLSConfig(&conf.TLSConfig)
+	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
 	}

discovery/registry.go

@@ -0,0 +1,257 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	configFieldPrefix      = "AUTO_DISCOVERY_"
+	staticConfigsKey       = "static_configs"
+	staticConfigsFieldName = configFieldPrefix + staticConfigsKey
+)
+
+var (
+	configNames      = make(map[string]Config)
+	configFieldNames = make(map[reflect.Type]string)
+	configFields     []reflect.StructField
+
+	configTypesMu sync.Mutex
+	configTypes   = make(map[reflect.Type]reflect.Type)
+
+	emptyStructType = reflect.TypeOf(struct{}{})
+	configsType     = reflect.TypeOf(Configs{})
+)
+
+// RegisterConfig registers the given Config type for YAML marshaling and unmarshaling.
+func RegisterConfig(config Config) {
+	registerConfig(config.Name()+"_sd_configs", reflect.TypeOf(config), config)
+}
+
+func init() {
+	// N.B.: static_configs is the only Config type implemented by default.
+	// All other types are registered at init by their implementing packages.
+	elemTyp := reflect.TypeOf(&targetgroup.Group{})
+	registerConfig(staticConfigsKey, elemTyp, StaticConfig{})
+}
+
+func registerConfig(yamlKey string, elemType reflect.Type, config Config) {
+	name := config.Name()
+	if _, ok := configNames[name]; ok {
+		panic(fmt.Sprintf("discovery: Config named %q is already registered", name))
+	}
+	configNames[name] = config
+
+	fieldName := configFieldPrefix + yamlKey // Field must be exported.
+	configFieldNames[elemType] = fieldName
+
+	// Insert fields in sorted order.
+	i := sort.Search(len(configFields), func(k int) bool {
+		return fieldName < configFields[k].Name
+	})
+	configFields = append(configFields, reflect.StructField{}) // Add empty field at end.
+	copy(configFields[i+1:], configFields[i:])                 // Shift fields to the right.
+	configFields[i] = reflect.StructField{                     // Write new field in place.
+		Name: fieldName,
+		Type: reflect.SliceOf(elemType),
+		Tag:  reflect.StructTag(`yaml:"` + yamlKey + `,omitempty"`),
+	}
+}
+
+func getConfigType(out reflect.Type) reflect.Type {
+	configTypesMu.Lock()
+	defer configTypesMu.Unlock()
+	if typ, ok := configTypes[out]; ok {
+		return typ
+	}
+	// Initial exported fields map one-to-one.
+	var fields []reflect.StructField
+	for i, n := 0, out.NumField(); i < n; i++ {
+		switch field := out.Field(i); {
+		case field.PkgPath == "" && field.Type != configsType:
+			fields = append(fields, field)
+		default:
+			fields = append(fields, reflect.StructField{
+				Name:    "_" + field.Name, // Field must be unexported.
+				PkgPath: out.PkgPath(),
+				Type:    emptyStructType,
+			})
+		}
+	}
+	// Append extra config fields on the end.
+	fields = append(fields, configFields...)
+	typ := reflect.StructOf(fields)
+	configTypes[out] = typ
+	return typ
+}
+
+// UnmarshalYAMLWithInlineConfigs helps implement yaml.Unmarshal for structs
+// that have a Configs field that should be inlined.
+func UnmarshalYAMLWithInlineConfigs(out interface{}, unmarshal func(interface{}) error) error {
+	outVal := reflect.ValueOf(out)
+	if outVal.Kind() != reflect.Ptr {
+		return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out)
+	}
+	outVal = outVal.Elem()
+	if outVal.Kind() != reflect.Struct {
+		return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out)
+	}
+	outTyp := outVal.Type()
+
+	cfgTyp := getConfigType(outTyp)
+	cfgPtr := reflect.New(cfgTyp)
+	cfgVal := cfgPtr.Elem()
+
+	// Copy shared fields (defaults) to dynamic value.
+	var configs *Configs
+	for i, n := 0, outVal.NumField(); i < n; i++ {
+		if outTyp.Field(i).Type == configsType {
+			configs = outVal.Field(i).Addr().Interface().(*Configs)
+			continue
+		}
+		if cfgTyp.Field(i).PkgPath != "" {
+			continue // Field is unexported: ignore.
+		}
+		cfgVal.Field(i).Set(outVal.Field(i))
+	}
+	if configs == nil {
+		return fmt.Errorf("discovery: Configs field not found in type: %T", out)
+	}
+
+	// Unmarshal into dynamic value.
+	if err := unmarshal(cfgPtr.Interface()); err != nil {
+		return replaceYAMLTypeError(err, cfgTyp, outTyp)
+	}
+
+	// Copy shared fields from dynamic value.
+	for i, n := 0, outVal.NumField(); i < n; i++ {
+		if cfgTyp.Field(i).PkgPath != "" {
+			continue // Field is unexported: ignore.
+		}
+		outVal.Field(i).Set(cfgVal.Field(i))
+	}
+
+	var err error
+	*configs, err = readConfigs(cfgVal, outVal.NumField())
+	return err
+}
+
+func readConfigs(structVal reflect.Value, startField int) (Configs, error) {
+	var (
+		configs Configs
+		targets []*targetgroup.Group
+	)
+	for i, n := startField, structVal.NumField(); i < n; i++ {
+		field := structVal.Field(i)
+		if field.Kind() != reflect.Slice {
+			panic("discovery: internal error: field is not a slice")
+		}
+		for k := 0; k < field.Len(); k++ {
+			val := field.Index(k)
+			if val.IsZero() || (val.Kind() == reflect.Ptr && val.Elem().IsZero()) {
+				key := configFieldNames[field.Type().Elem()]
+				key = strings.TrimPrefix(key, configFieldPrefix)
+				return nil, fmt.Errorf("empty or null section in %s", key)
+			}
+			switch c := val.Interface().(type) {
+			case *targetgroup.Group:
+				// Add index to the static config target groups for unique identification
+				// within scrape pool.
+				c.Source = strconv.Itoa(len(targets))
+				// Coalesce multiple static configs into a single static config.
+				targets = append(targets, c)
+			case Config:
+				configs = append(configs, c)
+			default:
+				panic("discovery: internal error: slice element is not a Config")
+			}
+		}
+	}
+	if len(targets) > 0 {
+		configs = append(configs, StaticConfig(targets))
+	}
+	return configs, nil
+}
+
+// MarshalYAMLWithInlineConfigs helps implement yaml.Marshal for structs
+// that have a Configs field that should be inlined.
+func MarshalYAMLWithInlineConfigs(in interface{}) (interface{}, error) {
+	inVal := reflect.ValueOf(in)
+	for inVal.Kind() == reflect.Ptr {
+		inVal = inVal.Elem()
+	}
+	inTyp := inVal.Type()
+
+	cfgTyp := getConfigType(inTyp)
+	cfgPtr := reflect.New(cfgTyp)
+	cfgVal := cfgPtr.Elem()
+
+	// Copy shared fields to dynamic value.
+	var configs *Configs
+	for i, n := 0, inTyp.NumField(); i < n; i++ {
+		if inTyp.Field(i).Type == configsType {
+			configs = inVal.Field(i).Addr().Interface().(*Configs)
+		}
+		if cfgTyp.Field(i).PkgPath != "" {
+			continue // Field is unexported: ignore.
+		}
+		cfgVal.Field(i).Set(inVal.Field(i))
+	}
+	if configs == nil {
+		return nil, fmt.Errorf("discovery: Configs field not found in type: %T", in)
+	}
+
+	if err := writeConfigs(cfgVal, *configs); err != nil {
+		return nil, err
+	}
+
+	return cfgPtr.Interface(), nil
+}
+
+func writeConfigs(structVal reflect.Value, configs Configs) error {
+	targets := structVal.FieldByName(staticConfigsFieldName).Addr().Interface().(*[]*targetgroup.Group)
+	for _, c := range configs {
+		if sc, ok := c.(StaticConfig); ok {
+			*targets = append(*targets, sc...)
+			continue
+		}
+		fieldName, ok := configFieldNames[reflect.TypeOf(c)]
+		if !ok {
+			return fmt.Errorf("discovery: cannot marshal unregistered Config type: %T", c)
+		}
+		field := structVal.FieldByName(fieldName)
+		field.Set(reflect.Append(field, reflect.ValueOf(c)))
+	}
+	return nil
+}
+
+func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error {
+	if e, ok := err.(*yaml.TypeError); ok {
+		oldStr := oldTyp.String()
+		newStr := newTyp.String()
+		for i, s := range e.Errors {
+			e.Errors[i] = strings.Replace(s, oldStr, newStr, -1)
+		}
+	}
+	return err
+}

discovery/triton/triton.go

@@ -27,9 +27,10 @@ import (
 	"github.com/go-kit/kit/log"
 	conntrack "github.com/mwitkow/go-conntrack"
 	"github.com/pkg/errors"
-	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
@@ -52,6 +53,10 @@ var DefaultSDConfig = SDConfig{
 	Version:         1,
 }
 
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
 // SDConfig is the configuration for Triton based service discovery.
 type SDConfig struct {
 	Account         string           `yaml:"account"`
@@ -61,10 +66,23 @@ type SDConfig struct {
 	Groups          []string         `yaml:"groups,omitempty"`
 	Port            int              `yaml:"port"`
 	RefreshInterval model.Duration   `yaml:"refresh_interval,omitempty"`
-	TLSConfig       config_util.TLSConfig `yaml:"tls_config,omitempty"`
+	TLSConfig       config.TLSConfig `yaml:"tls_config,omitempty"`
 	Version         int              `yaml:"version"`
 }
 
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "triton" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return New(opts.Logger, c)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.TLSConfig.SetDirectory(dir)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultSDConfig
@@ -122,7 +140,7 @@ type Discovery struct {
 
 // New returns a new Discovery which periodically refreshes its targets.
 func New(logger log.Logger, conf *SDConfig) (*Discovery, error) {
-	tls, err := config_util.NewTLSConfig(&conf.TLSConfig)
+	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
 	}

discovery/zookeeper/zookeeper.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/samuel/go-zookeeper/zk"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/prometheus/prometheus/util/treecache"
@@ -43,6 +44,11 @@ var (
 	}
 )
 
+func init() {
+	discovery.RegisterConfig(&ServersetSDConfig{})
+	discovery.RegisterConfig(&NerveSDConfig{})
+}
+
 // ServersetSDConfig is the configuration for Twitter serversets in Zookeeper based discovery.
 type ServersetSDConfig struct {
 	Servers []string       `yaml:"servers"`
@@ -50,6 +56,14 @@ type ServersetSDConfig struct {
 	Timeout model.Duration `yaml:"timeout,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*ServersetSDConfig) Name() string { return "serverset" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *ServersetSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewServersetDiscovery(c, opts.Logger)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *ServersetSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultServersetSDConfig
@@ -79,6 +93,14 @@ type NerveSDConfig struct {
 	Timeout model.Duration `yaml:"timeout,omitempty"`
 }
 
+// Name returns the name of the Config.
+func (*NerveSDConfig) Name() string { return "nerve" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *NerveSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewNerveDiscovery(c, opts.Logger)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *NerveSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultNerveSDConfig

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/prometheus/alertmanager v0.21.0
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.11.1
+	github.com/prometheus/common v0.13.0
 	github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20200627165143-92b8a710ab6c

go.sum

@@ -656,8 +656,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.11.1 h1:0ZISXCMRuCZcxF77aT1BXY5m74mX2vrGYl1dSwBI0Jo=
+github.com/prometheus/common v0.13.0 h1:vJlpe9wPgDRM1Z+7Wj3zUUjY1nr6/1jNKyl7llliccg=
-github.com/prometheus/common v0.11.1/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/common v0.13.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=

notifier/notifier.go

@@ -634,7 +634,7 @@ type alertmanagerSet struct {
 }
 
 func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger, metrics *alertMetrics) (*alertmanagerSet, error) {
-	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "alertmanager", false)
+	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "alertmanager", false, false)
 	if err != nil {
 		return nil, err
 	}

notifier/notifier_test.go

@@ -146,7 +146,7 @@ func TestHandlerSendAll(t *testing.T) {
 				Username: "prometheus",
 				Password: "testing_password",
 			},
-		}, "auth_alertmanager", false)
+		}, "auth_alertmanager", false, false)
 
 	h.alertmanagers = make(map[string]*alertmanagerSet)
 

scrape/scrape.go

@@ -219,7 +219,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, jitterSeed
 		logger = log.NewNopLogger()
 	}
 
-	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, false)
+	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, false, false)
 	if err != nil {
 		targetScrapePoolsFailed.Inc()
 		return nil, errors.Wrap(err, "error creating HTTP client")
@@ -321,7 +321,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 	sp.mtx.Lock()
 	defer sp.mtx.Unlock()
 
-	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, false)
+	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, false, false)
 	if err != nil {
 		targetScrapePoolReloadsFailed.Inc()
 		return errors.Wrap(err, "error creating HTTP client")

scrape/target_test.go

@@ -151,7 +151,7 @@ func TestNewHTTPBearerToken(t *testing.T) {
 	cfg := config_util.HTTPClientConfig{
 		BearerToken: "1234",
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +178,7 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
 	cfg := config_util.HTTPClientConfig{
 		BearerTokenFile: "testdata/bearertoken.txt",
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -207,7 +207,7 @@ func TestNewHTTPBasicAuth(t *testing.T) {
 			Password: "password123",
 		},
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -235,7 +235,7 @@ func TestNewHTTPCACert(t *testing.T) {
 			CAFile: caCertPath,
 		},
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,7 +268,7 @@ func TestNewHTTPClientCert(t *testing.T) {
 			KeyFile:  "testdata/client.key",
 		},
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -297,7 +297,7 @@ func TestNewHTTPWithServerName(t *testing.T) {
 			ServerName: "prometheus.rocks",
 		},
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -326,7 +326,7 @@ func TestNewHTTPWithBadServerName(t *testing.T) {
 			ServerName: "badname",
 		},
 	}
-	c, err := config_util.NewClientFromConfig(cfg, "test", false)
+	c, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -364,7 +364,7 @@ func TestNewClientWithBadTLSConfig(t *testing.T) {
 			KeyFile:  "testdata/nonexistent_client.key",
 		},
 	}
-	_, err := config_util.NewClientFromConfig(cfg, "test", false)
+	_, err := config_util.NewClientFromConfig(cfg, "test", false, false)
 	if err == nil {
 		t.Fatalf("Expected error, got nil.")
 	}

storage/remote/client.go

@@ -104,7 +104,7 @@ type ReadClient interface {
 
 // newReadClient creates a new client for remote read.
 func newReadClient(name string, conf *ClientConfig) (ReadClient, error) {
-	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_read_client", false)
+	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_read_client", false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -122,7 +122,7 @@ func newReadClient(name string, conf *ClientConfig) (ReadClient, error) {
 
 // NewWriteClient creates a new client for remote write.
 func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
-	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_write_client", false)
+	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_write_client", false, false)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/prometheus/common/config/config.go

@@ -16,6 +16,8 @@
 
 package config
 
+import "path/filepath"
+
 // Secret special type for storing secrets.
 type Secret string
 
@@ -32,3 +34,20 @@ func (s *Secret) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type plain Secret
 	return unmarshal((*plain)(s))
 }
+
+// DirectorySetter is a config type that contains file paths that may
+// be relative to the file containing the config.
+type DirectorySetter interface {
+	// SetDirectory joins any relative file paths with dir.
+	// Any paths that are empty or absolute remain unchanged.
+	SetDirectory(dir string)
+}
+
+// JoinDir joins dir and path if path is relative.
+// If path is empty or absolute, it is returned unchanged.
+func JoinDir(dir, path string) string {
+	if path == "" || filepath.IsAbs(path) {
+		return path
+	}
+	return filepath.Join(dir, path)
+}

vendor/github.com/prometheus/common/config/http_config.go

@@ -44,6 +44,14 @@ type BasicAuth struct {
 	PasswordFile string `yaml:"password_file,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (a *BasicAuth) SetDirectory(dir string) {
+	if a == nil {
+		return
+	}
+	a.PasswordFile = JoinDir(dir, a.PasswordFile)
+}
+
 // URL is a custom URL type that allows validation at configuration load time.
 type URL struct {
 	*url.URL
@@ -86,6 +94,16 @@ type HTTPClientConfig struct {
 	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *HTTPClientConfig) SetDirectory(dir string) {
+	if c == nil {
+		return
+	}
+	c.TLSConfig.SetDirectory(dir)
+	c.BasicAuth.SetDirectory(dir)
+	c.BearerTokenFile = JoinDir(dir, c.BearerTokenFile)
+}
+
 // Validate validates the HTTPClientConfig to check only one of BearerToken,
 // BasicAuth and BearerTokenFile is configured.
 func (c *HTTPClientConfig) Validate() error {
@@ -123,8 +141,8 @@ func newClient(rt http.RoundTripper) *http.Client {
 
 // NewClientFromConfig returns a new HTTP client configured for the
 // given config.HTTPClientConfig. The name is used as go-conntrack metric label.
-func NewClientFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives bool) (*http.Client, error) {
+func NewClientFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives, enableHTTP2 bool) (*http.Client, error) {
-	rt, err := NewRoundTripperFromConfig(cfg, name, disableKeepAlives)
+	rt, err := NewRoundTripperFromConfig(cfg, name, disableKeepAlives, enableHTTP2)
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +151,7 @@ func NewClientFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives bo
 
 // NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the
 // given config.HTTPClientConfig. The name is used as go-conntrack metric label.
-func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives bool) (http.RoundTripper, error) {
+func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives, enableHTTP2 bool) (http.RoundTripper, error) {
 	newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
 		// The only timeout we care about is the configured scrape timeout.
 		// It is applied on request. So we leave out any timings here.
@@ -154,11 +172,19 @@ func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, disableKeepAli
 				conntrack.DialWithName(name),
 			),
 		}
+		if enableHTTP2 {
+			// HTTP/2 support is golang has many problematic cornercases where
+			// dead connections would be kept and used in connection pools.
+			// https://github.com/golang/go/issues/32388
+			// https://github.com/golang/go/issues/39337
+			// https://github.com/golang/go/issues/39750
+			// TODO: Re-Enable HTTP/2 once upstream issue is fixed.
 			// TODO: use ForceAttemptHTTP2 when we move to Go 1.13+.
 			err := http2.ConfigureTransport(rt.(*http.Transport))
 			if err != nil {
 				return nil, err
 			}
+		}
 
 		// If a bearer token is provided, create a round tripper that will set the
 		// Authorization header correctly on each request.
@@ -344,6 +370,16 @@ type TLSConfig struct {
 	InsecureSkipVerify bool `yaml:"insecure_skip_verify"`
 }
 
+// SetDirectory joins any relative file paths with dir.
+func (c *TLSConfig) SetDirectory(dir string) {
+	if c == nil {
+		return
+	}
+	c.CAFile = JoinDir(dir, c.CAFile)
+	c.CertFile = JoinDir(dir, c.CertFile)
+	c.KeyFile = JoinDir(dir, c.KeyFile)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type plain TLSConfig

vendor/modules.txt

@@ -385,7 +385,7 @@ github.com/prometheus/client_golang/prometheus/testutil/promlint
 # github.com/prometheus/client_model v0.2.0
 ## explicit
 github.com/prometheus/client_model/go
-# github.com/prometheus/common v0.11.1
+# github.com/prometheus/common v0.13.0
 ## explicit
 github.com/prometheus/common/config
 github.com/prometheus/common/expfmt