Merge pull request #2215 from prometheus/alertingsd2

Discover Alertmanagers dynamically

cmd/prometheus/config.go

@@ -47,6 +47,7 @@ var cfg = struct {
 	storage            local.MemorySeriesStorageOptions
 	localStorageEngine string
 	notifier           notifier.Options
+	notifierTimeout    time.Duration
 	queryEngine        promql.EngineOptions
 	web                web.Options
 	remote             remote.Options
@@ -228,7 +229,7 @@ func init() {
 		"The capacity of the queue for pending alert manager notifications.",
 	)
 	cfg.fs.DurationVar(
-		&cfg.notifier.Timeout, "alertmanager.timeout", 10*time.Second,
+		&cfg.notifierTimeout, "alertmanager.timeout", 10*time.Second,
 		"Alert manager HTTP API timeout.",
 	)
 
@@ -283,7 +284,6 @@ func parse(args []string) error {
 		if err := validateAlertmanagerURL(u); err != nil {
 			return err
 		}
-		cfg.notifier.AlertmanagerURLs = cfg.alertmanagerURLs.slice()
 	}
 
 	cfg.remote.InfluxdbPassword = os.Getenv("INFLUXDB_PW")

cmd/prometheus/main.go

@@ -18,6 +18,7 @@ import (
 	"flag"
 	"fmt"
 	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
+	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
@@ -25,6 +26,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 	"golang.org/x/net/context"
 
@@ -126,6 +128,7 @@ func Main() int {
 	cfg.web.QueryEngine = queryEngine
 	cfg.web.TargetManager = targetManager
 	cfg.web.RuleManager = ruleManager
+	cfg.web.Notifier = notifier
 
 	cfg.web.Version = &web.PrometheusVersion{
 		Version:   version.Version,
@@ -259,6 +262,31 @@ func reloadConfig(filename string, rls ...Reloadable) (err error) {
 		return fmt.Errorf("couldn't load configuration (-config.file=%s): %v", filename, err)
 	}
 
+	// Add AlertmanagerConfigs for legacy Alertmanager URL flags.
+	for us := range cfg.alertmanagerURLs {
+		u, err := url.Parse(us)
+		if err != nil {
+			return err
+		}
+		acfg := &config.AlertmanagerConfig{
+			Scheme:     u.Scheme,
+			PathPrefix: u.Path,
+			Timeout:    cfg.notifierTimeout,
+			ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{
+				StaticConfigs: []*config.TargetGroup{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue(u.Host),
+							},
+						},
+					},
+				},
+			},
+		}
+		conf.AlertingConfig.AlertmanagerConfigs = append(conf.AlertingConfig.AlertmanagerConfigs, acfg)
+	}
+
 	failed := false
 	for _, rl := range rls {
 		if err := rl.ApplyConfig(conf); err != nil {

cmd/promtool/main.go

@@ -102,15 +102,15 @@ func checkConfig(t cli.Term, filename string) ([]string, error) {
 	}
 
 	for _, scfg := range cfg.ScrapeConfigs {
-		if err := checkFileExists(scfg.BearerTokenFile); err != nil {
+		if err := checkFileExists(scfg.HTTPClientConfig.BearerTokenFile); err != nil {
-			return nil, fmt.Errorf("error checking bearer token file %q: %s", scfg.BearerTokenFile, err)
+			return nil, fmt.Errorf("error checking bearer token file %q: %s", scfg.HTTPClientConfig.BearerTokenFile, err)
 		}
 
-		if err := checkTLSConfig(scfg.TLSConfig); err != nil {
+		if err := checkTLSConfig(scfg.HTTPClientConfig.TLSConfig); err != nil {
 			return nil, err
 		}
 
-		for _, kd := range scfg.KubernetesSDConfigs {
+		for _, kd := range scfg.ServiceDiscoveryConfig.KubernetesSDConfigs {
 			if err := checkTLSConfig(kd.TLSConfig); err != nil {
 				return nil, err
 			}

config/config.go

@@ -87,6 +87,12 @@ var (
 		HonorLabels: false,
 	}
 
+	// DefaultAlertmanagersConfig is the default alertmanager configuration.
+	DefaultAlertmanagersConfig = AlertmanagerConfig{
+		Scheme:  "http",
+		Timeout: 10 * time.Second,
+	}
+
 	// DefaultRelabelConfig is the default Relabel configuration.
 	DefaultRelabelConfig = RelabelConfig{
 		Action:      RelabelReplace,
@@ -214,25 +220,33 @@ func resolveFilepaths(baseDir string, cfg *Config) {
 		cfg.RuleFiles[i] = join(rf)
 	}
 
-	for _, scfg := range cfg.ScrapeConfigs {
+	clientPaths := func(scfg *HTTPClientConfig) {
 		scfg.BearerTokenFile = join(scfg.BearerTokenFile)
 		scfg.TLSConfig.CAFile = join(scfg.TLSConfig.CAFile)
 		scfg.TLSConfig.CertFile = join(scfg.TLSConfig.CertFile)
 		scfg.TLSConfig.KeyFile = join(scfg.TLSConfig.KeyFile)
-
+	}
-		for _, kcfg := range scfg.KubernetesSDConfigs {
+	sdPaths := func(cfg *ServiceDiscoveryConfig) {
+		for _, kcfg := range cfg.KubernetesSDConfigs {
 			kcfg.BearerTokenFile = join(kcfg.BearerTokenFile)
 			kcfg.TLSConfig.CAFile = join(kcfg.TLSConfig.CAFile)
 			kcfg.TLSConfig.CertFile = join(kcfg.TLSConfig.CertFile)
 			kcfg.TLSConfig.KeyFile = join(kcfg.TLSConfig.KeyFile)
 		}
-
+		for _, mcfg := range cfg.MarathonSDConfigs {
-		for _, mcfg := range scfg.MarathonSDConfigs {
 			mcfg.TLSConfig.CAFile = join(mcfg.TLSConfig.CAFile)
 			mcfg.TLSConfig.CertFile = join(mcfg.TLSConfig.CertFile)
 			mcfg.TLSConfig.KeyFile = join(mcfg.TLSConfig.KeyFile)
 		}
+	}
 
+	for _, cfg := range cfg.ScrapeConfigs {
+		clientPaths(&cfg.HTTPClientConfig)
+		sdPaths(&cfg.ServiceDiscoveryConfig)
+	}
+	for _, cfg := range cfg.AlertingConfig.AlertmanagerConfigs {
+		clientPaths(&cfg.HTTPClientConfig)
+		sdPaths(&cfg.ServiceDiscoveryConfig)
 	}
 }
 
@@ -312,11 +326,6 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// AlertingConfig configures alerting and alertmanager related configs
-type AlertingConfig struct {
-	AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
-}
-
 // GlobalConfig configures values that are used across other configuration
 // objects.
 type GlobalConfig struct {
@@ -404,33 +413,8 @@ func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// ScrapeConfig configures a scraping unit for Prometheus.
+// ServiceDiscoveryConfig configures lists of different service discovery mechanisms.
-type ScrapeConfig struct {
+type ServiceDiscoveryConfig struct {
-	// The job name to which the job label is set by default.
-	JobName string `yaml:"job_name"`
-	// Indicator whether the scraped metrics should remain unmodified.
-	HonorLabels bool `yaml:"honor_labels,omitempty"`
-	// A set of query parameters with which the target is scraped.
-	Params url.Values `yaml:"params,omitempty"`
-	// How frequently to scrape the targets of this scrape config.
-	ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
-	// The timeout for scraping targets of this config.
-	ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
-	// The HTTP resource path on which to fetch metrics from targets.
-	MetricsPath string `yaml:"metrics_path,omitempty"`
-	// The URL scheme with which to fetch metrics from targets.
-	Scheme string `yaml:"scheme,omitempty"`
-	// The HTTP basic authentication credentials for the targets.
-	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
-	// The bearer token for the targets.
-	BearerToken string `yaml:"bearer_token,omitempty"`
-	// The bearer token file for the targets.
-	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
-	// HTTP proxy server to use to connect to the targets.
-	ProxyURL URL `yaml:"proxy_url,omitempty"`
-	// TLSConfig to use to connect to the targets.
-	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`
-
 	// List of labeled target groups for this job.
 	StaticConfigs []*TargetGroup `yaml:"static_configs,omitempty"`
 	// List of DNS service discovery configurations.
@@ -454,6 +438,72 @@ type ScrapeConfig struct {
 	// List of Azure service discovery configurations.
 	AzureSDConfigs []*AzureSDConfig `yaml:"azure_sd_configs,omitempty"`
 
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *ServiceDiscoveryConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain ServiceDiscoveryConfig
+	if err := unmarshal((*plain)(c)); err != nil {
+		return err
+	}
+	if err := checkOverflow(c.XXX, "service discovery config"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// HTTPClientConfig configures an HTTP client.
+type HTTPClientConfig struct {
+	// The HTTP basic authentication credentials for the targets.
+	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
+	// The bearer token for the targets.
+	BearerToken string `yaml:"bearer_token,omitempty"`
+	// The bearer token file for the targets.
+	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
+	// HTTP proxy server to use to connect to the targets.
+	ProxyURL URL `yaml:"proxy_url,omitempty"`
+	// TLSConfig to use to connect to the targets.
+	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+func (c *HTTPClientConfig) validate() error {
+	if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 {
+		return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured")
+	}
+	if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) {
+		return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured")
+	}
+	return nil
+}
+
+// ScrapeConfig configures a scraping unit for Prometheus.
+type ScrapeConfig struct {
+	// The job name to which the job label is set by default.
+	JobName string `yaml:"job_name"`
+	// Indicator whether the scraped metrics should remain unmodified.
+	HonorLabels bool `yaml:"honor_labels,omitempty"`
+	// A set of query parameters with which the target is scraped.
+	Params url.Values `yaml:"params,omitempty"`
+	// How frequently to scrape the targets of this scrape config.
+	ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
+	// The timeout for scraping targets of this config.
+	ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
+	// The HTTP resource path on which to fetch metrics from targets.
+	MetricsPath string `yaml:"metrics_path,omitempty"`
+	// The URL scheme with which to fetch metrics from targets.
+	Scheme string `yaml:"scheme,omitempty"`
+
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+
+	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig       `yaml:",inline"`
+
 	// List of target relabel configurations.
 	RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`
 	// List of metric relabel configurations.
@@ -477,15 +527,17 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if len(c.JobName) == 0 {
 		return fmt.Errorf("job_name is empty")
 	}
-	if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 {
+
-		return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured")
+	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
-	}
+	// We cannot make it a pointer as the parser panics for inlined pointer structs.
-	if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) {
+	// Thus we just do its validation here.
-		return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured")
+	if err := c.HTTPClientConfig.validate(); err != nil {
+		return err
 	}
+
 	// Check for users putting URLs in target groups.
 	if len(c.RelabelConfigs) == 0 {
-		for _, tg := range c.StaticConfigs {
+		for _, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
 			for _, t := range tg.Targets {
 				if err = CheckTargetAddress(t[model.AddressLabel]); err != nil {
 					return err
@@ -496,6 +548,83 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+// AlertingConfig configures alerting and alertmanager related configs
+type AlertingConfig struct {
+	AlertRelabelConfigs []*RelabelConfig      `yaml:"alert_relabel_configs,omitempty"`
+	AlertmanagerConfigs []*AlertmanagerConfig `yaml:"alertmanagers,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Create a clean global config as the previous one was already populated
+	// by the default due to the YAML parser behavior for empty blocks.
+	*c = AlertingConfig{}
+	type plain AlertingConfig
+	if err := unmarshal((*plain)(c)); err != nil {
+		return err
+	}
+	if err := checkOverflow(c.XXX, "alerting config"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// AlertmanagersConfig configures how Alertmanagers can be discovered and communicated with.
+type AlertmanagerConfig struct {
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+
+	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig       `yaml:",inline"`
+
+	// The URL scheme to use when talking to Alertmanagers.
+	Scheme string `yaml:"scheme,omitempty"`
+	// Path prefix to add in front of the push endpoint path.
+	PathPrefix string `yaml:"path_prefix,omitempty"`
+	// The timeout used when sending alerts.
+	Timeout time.Duration `yaml:"timeout,omitempty"`
+
+	// List of Alertmanager relabel configurations.
+	RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultAlertmanagersConfig
+	type plain AlertmanagerConfig
+	if err := unmarshal((*plain)(c)); err != nil {
+		return err
+	}
+	if err := checkOverflow(c.XXX, "alertmanager config"); err != nil {
+		return err
+	}
+
+	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
+	// We cannot make it a pointer as the parser panics for inlined pointer structs.
+	// Thus we just do its validation here.
+	if err := c.HTTPClientConfig.validate(); err != nil {
+		return err
+	}
+
+	// Check for users putting URLs in target groups.
+	if len(c.RelabelConfigs) == 0 {
+		for _, tg := range c.ServiceDiscoveryConfig.StaticConfigs {
+			for _, t := range tg.Targets {
+				if err := CheckTargetAddress(t[model.AddressLabel]); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 // CheckTargetAddress checks if target address is valid.
 func CheckTargetAddress(address model.LabelValue) error {
 	// For now check for a URL, we may want to expand this later.

config/config_test.go

@@ -68,29 +68,33 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			BearerTokenFile: "testdata/valid_token_file",
+			HTTPClientConfig: HTTPClientConfig{
-
+				BearerTokenFile: "testdata/valid_token_file",
-			StaticConfigs: []*TargetGroup{
-				{
-					Targets: []model.LabelSet{
-						{model.AddressLabel: "localhost:9090"},
-						{model.AddressLabel: "localhost:9191"},
-					},
-					Labels: model.LabelSet{
-						"my":   "label",
-						"your": "label",
-					},
-				},
 			},
 
-			FileSDConfigs: []*FileSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				StaticConfigs: []*TargetGroup{
-					Files:           []string{"foo/*.slow.json", "foo/*.slow.yml", "single/file.yml"},
+					{
-					RefreshInterval: model.Duration(10 * time.Minute),
+						Targets: []model.LabelSet{
+							{model.AddressLabel: "localhost:9090"},
+							{model.AddressLabel: "localhost:9191"},
+						},
+						Labels: model.LabelSet{
+							"my":   "label",
+							"your": "label",
+						},
+					},
 				},
-				{
+
-					Files:           []string{"bar/*.yaml"},
+				FileSDConfigs: []*FileSDConfig{
-					RefreshInterval: model.Duration(5 * time.Minute),
+					{
+						Files:           []string{"foo/*.slow.json", "foo/*.slow.yml", "single/file.yml"},
+						RefreshInterval: model.Duration(10 * time.Minute),
+					},
+					{
+						Files:           []string{"bar/*.yaml"},
+						RefreshInterval: model.Duration(5 * time.Minute),
+					},
 				},
 			},
 
@@ -130,28 +134,32 @@ var expectedConf = &Config{
 			ScrapeInterval: model.Duration(50 * time.Second),
 			ScrapeTimeout:  model.Duration(5 * time.Second),
 
-			BasicAuth: &BasicAuth{
+			HTTPClientConfig: HTTPClientConfig{
-				Username: "admin_name",
+				BasicAuth: &BasicAuth{
-				Password: "admin_password",
+					Username: "admin_name",
+					Password: "admin_password",
+				},
 			},
 			MetricsPath: "/my_path",
 			Scheme:      "https",
 
-			DNSSDConfigs: []*DNSSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				DNSSDConfigs: []*DNSSDConfig{
-					Names: []string{
+					{
-						"first.dns.address.domain.com",
+						Names: []string{
-						"second.dns.address.domain.com",
+							"first.dns.address.domain.com",
+							"second.dns.address.domain.com",
+						},
+						RefreshInterval: model.Duration(15 * time.Second),
+						Type:            "SRV",
 					},
-					RefreshInterval: model.Duration(15 * time.Second),
+					{
-					Type:            "SRV",
+						Names: []string{
-				},
+							"first.dns.address.domain.com",
-				{
+						},
-					Names: []string{
+						RefreshInterval: model.Duration(30 * time.Second),
-						"first.dns.address.domain.com",
+						Type:            "SRV",
 					},
-					RefreshInterval: model.Duration(30 * time.Second),
-					Type:            "SRV",
 				},
 			},
 
@@ -205,12 +213,14 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			ConsulSDConfigs: []*ConsulSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				ConsulSDConfigs: []*ConsulSDConfig{
-					Server:       "localhost:1234",
+					{
-					Services:     []string{"nginx", "cache", "mysql"},
+						Server:       "localhost:1234",
-					TagSeparator: DefaultConsulSDConfig.TagSeparator,
+						Services:     []string{"nginx", "cache", "mysql"},
-					Scheme:       DefaultConsulSDConfig.Scheme,
+						TagSeparator: DefaultConsulSDConfig.TagSeparator,
+						Scheme:       DefaultConsulSDConfig.Scheme,
+					},
 				},
 			},
 
@@ -234,12 +244,14 @@ var expectedConf = &Config{
 			MetricsPath: "/metrics",
 			Scheme:      "http",
 
-			TLSConfig: TLSConfig{
+			HTTPClientConfig: HTTPClientConfig{
-				CertFile: "testdata/valid_cert_file",
+				TLSConfig: TLSConfig{
-				KeyFile:  "testdata/valid_key_file",
+					CertFile: "testdata/valid_cert_file",
-			},
+					KeyFile:  "testdata/valid_key_file",
+				},
 
-			BearerToken: "avalidtoken",
+				BearerToken: "avalidtoken",
+			},
 		},
 		{
 			JobName: "service-kubernetes",
@@ -250,13 +262,15 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			KubernetesSDConfigs: []*KubernetesSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				KubernetesSDConfigs: []*KubernetesSDConfig{
-					APIServer: kubernetesSDHostURL(),
+					{
-					Role:      KubernetesRoleEndpoint,
+						APIServer: kubernetesSDHostURL(),
-					BasicAuth: &BasicAuth{
+						Role:      KubernetesRoleEndpoint,
-						Username: "myusername",
+						BasicAuth: &BasicAuth{
-						Password: "mypassword",
+							Username: "myusername",
+							Password: "mypassword",
+						},
 					},
 				},
 			},
@@ -270,16 +284,18 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			MarathonSDConfigs: []*MarathonSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				MarathonSDConfigs: []*MarathonSDConfig{
-					Servers: []string{
+					{
-						"https://marathon.example.com:443",
+						Servers: []string{
-					},
+							"https://marathon.example.com:443",
-					Timeout:         model.Duration(30 * time.Second),
+						},
-					RefreshInterval: model.Duration(30 * time.Second),
+						Timeout:         model.Duration(30 * time.Second),
-					TLSConfig: TLSConfig{
+						RefreshInterval: model.Duration(30 * time.Second),
-						CertFile: "testdata/valid_cert_file",
+						TLSConfig: TLSConfig{
-						KeyFile:  "testdata/valid_key_file",
+							CertFile: "testdata/valid_cert_file",
+							KeyFile:  "testdata/valid_key_file",
+						},
 					},
 				},
 			},
@@ -293,14 +309,16 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			EC2SDConfigs: []*EC2SDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				EC2SDConfigs: []*EC2SDConfig{
-					Region:          "us-east-1",
+					{
-					AccessKey:       "access",
+						Region:          "us-east-1",
-					SecretKey:       "secret",
+						AccessKey:       "access",
-					Profile:         "profile",
+						SecretKey:       "secret",
-					RefreshInterval: model.Duration(60 * time.Second),
+						Profile:         "profile",
-					Port:            80,
+						RefreshInterval: model.Duration(60 * time.Second),
+						Port:            80,
+					},
 				},
 			},
 		},
@@ -313,14 +331,16 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			AzureSDConfigs: []*AzureSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				AzureSDConfigs: []*AzureSDConfig{
-					SubscriptionID:  "11AAAA11-A11A-111A-A111-1111A1111A11",
+					{
-					TenantID:        "BBBB222B-B2B2-2B22-B222-2BB2222BB2B2",
+						SubscriptionID:  "11AAAA11-A11A-111A-A111-1111A1111A11",
-					ClientID:        "333333CC-3C33-3333-CCC3-33C3CCCCC33C",
+						TenantID:        "BBBB222B-B2B2-2B22-B222-2BB2222BB2B2",
-					ClientSecret:    "nAdvAK2oBuVym4IXix",
+						ClientID:        "333333CC-3C33-3333-CCC3-33C3CCCCC33C",
-					RefreshInterval: model.Duration(5 * time.Minute),
+						ClientSecret:    "nAdvAK2oBuVym4IXix",
-					Port:            9100,
+						RefreshInterval: model.Duration(5 * time.Minute),
+						Port:            9100,
+					},
 				},
 			},
 		},
@@ -333,11 +353,13 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			NerveSDConfigs: []*NerveSDConfig{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				NerveSDConfigs: []*NerveSDConfig{
-					Servers: []string{"localhost"},
+					{
-					Paths:   []string{"/monitoring"},
+						Servers: []string{"localhost"},
-					Timeout: model.Duration(10 * time.Second),
+						Paths:   []string{"/monitoring"},
+						Timeout: model.Duration(10 * time.Second),
+					},
 				},
 			},
 		},
@@ -350,10 +372,12 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			StaticConfigs: []*TargetGroup{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				StaticConfigs: []*TargetGroup{
-					Targets: []model.LabelSet{
+					{
-						{model.AddressLabel: "localhost:9090"},
+						Targets: []model.LabelSet{
+							{model.AddressLabel: "localhost:9090"},
+						},
 					},
 				},
 			},
@@ -367,10 +391,31 @@ var expectedConf = &Config{
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
 
-			StaticConfigs: []*TargetGroup{
+			ServiceDiscoveryConfig: ServiceDiscoveryConfig{
-				{
+				StaticConfigs: []*TargetGroup{
-					Targets: []model.LabelSet{
+					{
-						{model.AddressLabel: "localhost:9090"},
+						Targets: []model.LabelSet{
+							{model.AddressLabel: "localhost:9090"},
+						},
+					},
+				},
+			},
+		},
+	},
+	AlertingConfig: AlertingConfig{
+		AlertmanagerConfigs: []*AlertmanagerConfig{
+			{
+				Scheme:  "https",
+				Timeout: 10 * time.Second,
+				ServiceDiscoveryConfig: ServiceDiscoveryConfig{
+					StaticConfigs: []*TargetGroup{
+						{
+							Targets: []model.LabelSet{
+								{model.AddressLabel: "1.2.3.4:9093"},
+								{model.AddressLabel: "1.2.3.5:9093"},
+								{model.AddressLabel: "1.2.3.6:9093"},
+							},
+						},
 					},
 				},
 			},

config/testdata/conf.good.yml

@@ -174,3 +174,12 @@ scrape_configs:
   static_configs:
     - targets:
       - localhost:9090
+
+alerting:
+  alertmanagers:
+  - scheme: https
+    static_configs:
+    - targets:
+      - "1.2.3.4:9093"
+      - "1.2.3.5:9093"
+      - "1.2.3.6:9093"

discovery/discovery.go

@@ -49,7 +49,7 @@ type TargetProvider interface {
 }
 
 // ProvidersFromConfig returns all TargetProviders configured in cfg.
-func ProvidersFromConfig(cfg *config.ScrapeConfig) map[string]TargetProvider {
+func ProvidersFromConfig(cfg config.ServiceDiscoveryConfig) map[string]TargetProvider {
 	providers := map[string]TargetProvider{}
 
 	app := func(mech string, i int, tp TargetProvider) {

discovery/discovery_test.go

@@ -33,15 +33,14 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 		}
 	}
 
-	scrapeConfig := &config.ScrapeConfig{}
+	cfg := &config.ServiceDiscoveryConfig{}
 
 	sOne := `
-job_name: "foo"
 static_configs:
 - targets: ["foo:9090"]
 - targets: ["bar:9090"]
 `
-	if err := yaml.Unmarshal([]byte(sOne), scrapeConfig); err != nil {
+	if err := yaml.Unmarshal([]byte(sOne), cfg); err != nil {
 		t.Fatalf("Unable to load YAML config sOne: %s", err)
 	}
 	called := make(chan struct{})
@@ -54,22 +53,21 @@ static_configs:
 
 	go ts.Run(ctx)
 
-	ts.UpdateProviders(ProvidersFromConfig(scrapeConfig))
+	ts.UpdateProviders(ProvidersFromConfig(*cfg))
 	<-called
 
 	verifyPresence(ts.tgroups, "static/0/0", true)
 	verifyPresence(ts.tgroups, "static/0/1", true)
 
 	sTwo := `
-job_name: "foo"
 static_configs:
 - targets: ["foo:9090"]
 `
-	if err := yaml.Unmarshal([]byte(sTwo), scrapeConfig); err != nil {
+	if err := yaml.Unmarshal([]byte(sTwo), cfg); err != nil {
 		t.Fatalf("Unable to load YAML config sTwo: %s", err)
 	}
 
-	ts.UpdateProviders(ProvidersFromConfig(scrapeConfig))
+	ts.UpdateProviders(ProvidersFromConfig(*cfg))
 	<-called
 
 	verifyPresence(ts.tgroups, "static/0/0", true)

notifier/notifier.go

@@ -17,7 +17,10 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
+	"net/url"
+	"path"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -30,7 +33,9 @@ import (
 	"golang.org/x/net/context/ctxhttp"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/relabel"
+	"github.com/prometheus/prometheus/retrieval"
 )
 
 const (
@@ -62,15 +67,16 @@ type Notifier struct {
 	dropped       prometheus.Counter
 	queueLength   prometheus.Gauge
 	queueCapacity prometheus.Metric
+
+	alertmanagers   []*alertmanagerSet
+	cancelDiscovery func()
 }
 
 // Options are the configurable parameters of a Handler.
 type Options struct {
-	AlertmanagerURLs []string
+	QueueCapacity  int
-	QueueCapacity    int
+	ExternalLabels model.LabelSet
-	Timeout          time.Duration
+	RelabelConfigs []*config.RelabelConfig
-	ExternalLabels   model.LabelSet
-	RelabelConfigs   []*config.RelabelConfig
 }
 
 // New constructs a new Notifier.
@@ -139,6 +145,31 @@ func (n *Notifier) ApplyConfig(conf *config.Config) error {
 
 	n.opts.ExternalLabels = conf.GlobalConfig.ExternalLabels
 	n.opts.RelabelConfigs = conf.AlertingConfig.AlertRelabelConfigs
+
+	amSets := []*alertmanagerSet{}
+	ctx, cancel := context.WithCancel(n.ctx)
+
+	for _, cfg := range conf.AlertingConfig.AlertmanagerConfigs {
+		ams, err := newAlertmanagerSet(cfg)
+		if err != nil {
+			return err
+		}
+		amSets = append(amSets, ams)
+	}
+
+	// After all sets were created successfully, start them and cancel the
+	// old ones.
+	for _, ams := range amSets {
+		go ams.ts.Run(ctx)
+		ams.ts.UpdateProviders(discovery.ProvidersFromConfig(ams.cfg.ServiceDiscoveryConfig))
+	}
+	if n.cancelDiscovery != nil {
+		n.cancelDiscovery()
+	}
+
+	n.cancelDiscovery = cancel
+	n.alertmanagers = amSets
+
 	return nil
 }
 
@@ -170,13 +201,6 @@ func (n *Notifier) nextBatch() []*model.Alert {
 
 // Run dispatches notifications continuously.
 func (n *Notifier) Run() {
-	numAMs := len(n.opts.AlertmanagerURLs)
-	// Just warn once in the beginning to prevent noisy logs.
-	if numAMs == 0 {
-		log.Warnf("No AlertManagers configured, not dispatching any alerts")
-		return
-	}
-
 	for {
 		select {
 		case <-n.ctx.Done():
@@ -185,17 +209,7 @@ func (n *Notifier) Run() {
 		}
 		alerts := n.nextBatch()
 
-		if numAMs > 0 {
+		if !n.sendAll(alerts...) {
-
-			if len(alerts) > 0 {
-				numErrors := n.sendAll(alerts...)
-				// Increment the dropped counter if we could not send
-				// successfully to a single AlertManager.
-				if numErrors == numAMs {
-					n.dropped.Add(float64(len(alerts)))
-				}
-			}
-		} else {
 			n.dropped.Add(float64(len(alerts)))
 		}
 		// If the queue still has items left, kick off the next iteration.
@@ -267,58 +281,73 @@ func (n *Notifier) setMore() {
 	}
 }
 
-func postURL(u string) string {
+// Alertmanagers returns a list Alertmanager URLs.
-	return strings.TrimRight(u, "/") + alertPushEndpoint
+func (n *Notifier) Alertmanagers() []string {
+	n.mtx.RLock()
+	amSets := n.alertmanagers
+	n.mtx.RUnlock()
+
+	var res []string
+
+	for _, ams := range amSets {
+		ams.mtx.RLock()
+		for _, am := range ams.ams {
+			res = append(res, am.url())
+		}
+		ams.mtx.RUnlock()
+	}
+
+	return res
 }
 
-// sendAll sends the alerts to all configured Alertmanagers at concurrently.
+// sendAll sends the alerts to all configured Alertmanagers concurrently.
-// It returns the number of sends that have failed.
+// It returns true if the alerts could be sent successfully to at least one Alertmanager.
-func (n *Notifier) sendAll(alerts ...*model.Alert) int {
+func (n *Notifier) sendAll(alerts ...*model.Alert) bool {
 	begin := time.Now()
 
 	b, err := json.Marshal(alerts)
 	if err != nil {
 		log.Errorf("Encoding alerts failed: %s", err)
-		return len(n.opts.AlertmanagerURLs)
+		return false
 	}
-	ctx, _ := context.WithTimeout(context.Background(), n.opts.Timeout)
 
-	send := func(u string) error {
+	n.mtx.RLock()
-		resp, err := ctxhttp.Post(ctx, http.DefaultClient, postURL(u), contentTypeJSON, bytes.NewReader(b))
+	amSets := n.alertmanagers
-		if err != nil {
+	n.mtx.RUnlock()
-			return err
-		}
-		defer resp.Body.Close()
-
-		// Any HTTP status 2xx is OK.
-		if resp.StatusCode/100 != 2 {
-			return fmt.Errorf("bad response status %v", resp.Status)
-		}
-		return err
-	}
 
 	var (
-		wg        sync.WaitGroup
+		wg         sync.WaitGroup
-		numErrors uint64
+		numSuccess uint64
 	)
-	for _, u := range n.opts.AlertmanagerURLs {
+	for _, ams := range amSets {
-		wg.Add(1)
+		ams.mtx.RLock()
 
-		go func(u string) {
+		for _, am := range ams.ams {
-			if err := send(u); err != nil {
+			wg.Add(1)
-				log.With("alertmanager", u).With("count", fmt.Sprintf("%d", len(alerts))).Errorf("Error sending alerts: %s", err)
-				n.errors.WithLabelValues(u).Inc()
-				atomic.AddUint64(&numErrors, 1)
-			}
-			n.latency.WithLabelValues(u).Observe(time.Since(begin).Seconds())
-			n.sent.WithLabelValues(u).Add(float64(len(alerts)))
 
-			wg.Done()
+			ctx, cancel := context.WithTimeout(n.ctx, ams.cfg.Timeout)
-		}(u)
+			defer cancel()
+
+			go func(am alertmanager) {
+				u := am.url()
+
+				if err := am.send(ctx, ams.client, b); err != nil {
+					log.With("alertmanager", u).With("count", len(alerts)).Errorf("Error sending alerts: %s", err)
+					n.errors.WithLabelValues(u).Inc()
+				} else {
+					atomic.AddUint64(&numSuccess, 1)
+				}
+				n.latency.WithLabelValues(u).Observe(time.Since(begin).Seconds())
+				n.sent.WithLabelValues(u).Add(float64(len(alerts)))
+
+				wg.Done()
+			}(am)
+		}
+		ams.mtx.RUnlock()
 	}
 	wg.Wait()
 
-	return int(numErrors)
+	return numSuccess > 0
 }
 
 // Stop shuts down the notification handler.
@@ -350,3 +379,160 @@ func (n *Notifier) Collect(ch chan<- prometheus.Metric) {
 	ch <- n.queueLength
 	ch <- n.queueCapacity
 }
+
+// alertmanager holds Alertmanager endpoint information.
+type alertmanager struct {
+	plainURL string // test injection hook
+	labels   model.LabelSet
+}
+
+const pathLabel = "__alerts_path__"
+
+func (a alertmanager) url() string {
+	if a.plainURL != "" {
+		return a.plainURL
+	}
+	u := &url.URL{
+		Scheme: string(a.labels[model.SchemeLabel]),
+		Host:   string(a.labels[model.AddressLabel]),
+		Path:   string(a.labels[pathLabel]),
+	}
+	return u.String()
+}
+
+func (a alertmanager) send(ctx context.Context, c *http.Client, b []byte) error {
+	resp, err := ctxhttp.Post(ctx, c, a.url(), contentTypeJSON, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Any HTTP status 2xx is OK.
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("bad response status %v", resp.Status)
+	}
+	return err
+}
+
+// alertmanagerSet contains a set of Alertmanagers discovered via a group of service
+// discovery definitions that have a common configuration on how alerts should be sent.
+type alertmanagerSet struct {
+	ts     *discovery.TargetSet
+	cfg    *config.AlertmanagerConfig
+	client *http.Client
+
+	mtx sync.RWMutex
+	ams []alertmanager
+}
+
+func newAlertmanagerSet(cfg *config.AlertmanagerConfig) (*alertmanagerSet, error) {
+	client, err := retrieval.NewHTTPClient(cfg.HTTPClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	s := &alertmanagerSet{
+		client: client,
+		cfg:    cfg,
+	}
+	s.ts = discovery.NewTargetSet(s)
+
+	return s, nil
+}
+
+// Sync extracts a deduplicated set of Alertmanager endpoints from a list
+// of target groups definitions.
+func (s *alertmanagerSet) Sync(tgs []*config.TargetGroup) {
+	all := []alertmanager{}
+
+	for _, tg := range tgs {
+		ams, err := alertmanagerFromGroup(tg, s.cfg)
+		if err != nil {
+			log.With("err", err).Error("generating discovered Alertmanagers failed")
+			continue
+		}
+		all = append(all, ams...)
+	}
+
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+	// Set new Alertmanagers and deduplicate them along their unique URL.
+	s.ams = []alertmanager{}
+	seen := map[string]struct{}{}
+
+	for _, am := range all {
+		us := am.url()
+		if _, ok := seen[us]; ok {
+			continue
+		}
+
+		seen[us] = struct{}{}
+		s.ams = append(s.ams, am)
+	}
+}
+
+func postPath(pre string) string {
+	return path.Join("/", pre, alertPushEndpoint)
+}
+
+// alertmanagersFromGroup extracts a list of alertmanagers from a target group and an associcated
+// AlertmanagerConfig.
+func alertmanagerFromGroup(tg *config.TargetGroup, cfg *config.AlertmanagerConfig) ([]alertmanager, error) {
+	var res []alertmanager
+
+	for _, lset := range tg.Targets {
+		// Set configured scheme as the initial scheme label for overwrite.
+		lset[model.SchemeLabel] = model.LabelValue(cfg.Scheme)
+		lset[pathLabel] = model.LabelValue(postPath(cfg.PathPrefix))
+
+		// Combine target labels with target group labels.
+		for ln, lv := range tg.Labels {
+			if _, ok := lset[ln]; !ok {
+				lset[ln] = lv
+			}
+		}
+		lset := relabel.Process(lset, cfg.RelabelConfigs...)
+		if lset == nil {
+			continue
+		}
+
+		// addPort checks whether we should add a default port to the address.
+		// If the address is not valid, we don't append a port either.
+		addPort := func(s string) bool {
+			// If we can split, a port exists and we don't have to add one.
+			if _, _, err := net.SplitHostPort(s); err == nil {
+				return false
+			}
+			// If adding a port makes it valid, the previous error
+			// was not due to an invalid address and we can append a port.
+			_, _, err := net.SplitHostPort(s + ":1234")
+			return err == nil
+		}
+		// If it's an address with no trailing port, infer it based on the used scheme.
+		if addr := string(lset[model.AddressLabel]); addPort(addr) {
+			// Addresses reaching this point are already wrapped in [] if necessary.
+			switch lset[model.SchemeLabel] {
+			case "http", "":
+				addr = addr + ":80"
+			case "https":
+				addr = addr + ":443"
+			default:
+				return nil, fmt.Errorf("invalid scheme: %q", cfg.Scheme)
+			}
+			lset[model.AddressLabel] = model.LabelValue(addr)
+		}
+		if err := config.CheckTargetAddress(lset[model.AddressLabel]); err != nil {
+			return nil, err
+		}
+
+		// Meta labels are deleted after relabelling. Other internal labels propagate to
+		// the target which decides whether they will be part of their label set.
+		for ln := range lset {
+			if strings.HasPrefix(string(ln), model.MetaLabelPrefix) {
+				delete(lset, ln)
+			}
+		}
+
+		res = append(res, alertmanager{labels: lset})
+	}
+	return res, nil
+}

notifier/notifier_test.go

@@ -27,34 +27,34 @@ import (
 	"github.com/prometheus/prometheus/config"
 )
 
-func TestPostURL(t *testing.T) {
+func TestPostPath(t *testing.T) {
 	var cases = []struct {
 		in, out string
 	}{
 		{
-			in:  "http://localhost:9093",
+			in:  "",
-			out: "http://localhost:9093/api/v1/alerts",
+			out: "/api/v1/alerts",
 		},
 		{
-			in:  "http://localhost:9093/",
+			in:  "/",
-			out: "http://localhost:9093/api/v1/alerts",
+			out: "/api/v1/alerts",
 		},
 		{
-			in:  "http://localhost:9093/prefix",
+			in:  "/prefix",
-			out: "http://localhost:9093/prefix/api/v1/alerts",
+			out: "/prefix/api/v1/alerts",
 		},
 		{
-			in:  "http://localhost:9093/prefix//",
+			in:  "/prefix//",
-			out: "http://localhost:9093/prefix/api/v1/alerts",
+			out: "/prefix/api/v1/alerts",
 		},
 		{
-			in:  "http://localhost:9093/prefix//",
+			in:  "prefix//",
-			out: "http://localhost:9093/prefix/api/v1/alerts",
+			out: "/prefix/api/v1/alerts",
 		},
 	}
 	for _, c := range cases {
-		if res := postURL(c.in); res != c.out {
+		if res := postPath(c.in); res != c.out {
-			t.Errorf("Expected post URL %q for %q but got %q", c.out, c.in, res)
+			t.Errorf("Expected post path %q for %q but got %q", c.out, c.in, res)
 		}
 	}
 }
@@ -123,9 +123,6 @@ func TestHandlerSendAll(t *testing.T) {
 	)
 
 	f := func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path != alertPushEndpoint {
-			t.Fatalf("Bad endpoint %q used, expected %q", r.URL.Path, alertPushEndpoint)
-		}
 		defer r.Body.Close()
 
 		var alerts model.Alerts
@@ -150,9 +147,15 @@ func TestHandlerSendAll(t *testing.T) {
 	defer server1.Close()
 	defer server2.Close()
 
-	h := New(&Options{
+	h := New(&Options{})
-		AlertmanagerURLs: []string{server1.URL, server2.URL},
+	h.alertmanagers = append(h.alertmanagers, &alertmanagerSet{
-		Timeout:          time.Minute,
+		ams: []alertmanager{
+			{plainURL: server1.URL},
+			{plainURL: server2.URL},
+		},
+		cfg: &config.AlertmanagerConfig{
+			Timeout: time.Second,
+		},
 	})
 
 	for i := range make([]struct{}, maxBatchSize) {
@@ -170,18 +173,18 @@ func TestHandlerSendAll(t *testing.T) {
 
 	status1 = http.StatusOK
 	status2 = http.StatusOK
-	if ne := h.sendAll(h.queue...); ne != 0 {
+	if !h.sendAll(h.queue...) {
-		t.Fatalf("Unexpected number of failed sends: %d", ne)
+		t.Fatalf("all sends failed unexpectedly")
 	}
 
 	status1 = http.StatusNotFound
-	if ne := h.sendAll(h.queue...); ne != 1 {
+	if !h.sendAll(h.queue...) {
-		t.Fatalf("Unexpected number of failed sends: %d", ne)
+		t.Fatalf("all sends failed unexpectedly")
 	}
 
 	status2 = http.StatusInternalServerError
-	if ne := h.sendAll(h.queue...); ne != 2 {
+	if h.sendAll(h.queue...) {
-		t.Fatalf("Unexpected number of failed sends: %d", ne)
+		t.Fatalf("all sends succeeded unexpectedly")
 	}
 }
 
@@ -305,9 +308,15 @@ func TestHandlerQueueing(t *testing.T) {
 	}))
 
 	h := New(&Options{
-		AlertmanagerURLs: []string{server.URL},
+		QueueCapacity: 3 * maxBatchSize,
-		Timeout:          time.Second,
+	})
-		QueueCapacity:    3 * maxBatchSize,
+	h.alertmanagers = append(h.alertmanagers, &alertmanagerSet{
+		ams: []alertmanager{
+			{plainURL: server.URL},
+		},
+		cfg: &config.AlertmanagerConfig{
+			Timeout: time.Second,
+		},
 	})
 
 	var alerts model.Alerts

retrieval/scrape.go

@@ -106,7 +106,7 @@ type scrapePool struct {
 }
 
 func newScrapePool(ctx context.Context, cfg *config.ScrapeConfig, app storage.SampleAppender) *scrapePool {
-	client, err := NewHTTPClient(cfg)
+	client, err := NewHTTPClient(cfg.HTTPClientConfig)
 	if err != nil {
 		// Any errors that could occur here should be caught during config validation.
 		log.Errorf("Error creating HTTP client for job %q: %s", cfg.JobName, err)
@@ -153,7 +153,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) {
 	sp.mtx.Lock()
 	defer sp.mtx.Unlock()
 
-	client, err := NewHTTPClient(cfg)
+	client, err := NewHTTPClient(cfg.HTTPClientConfig)
 	if err != nil {
 		// Any errors that could occur here should be caught during config validation.
 		log.Errorf("Error creating HTTP client for job %q: %s", cfg.JobName, err)

retrieval/target.go

@@ -68,7 +68,7 @@ func NewTarget(labels, metaLabels model.LabelSet, params url.Values) *Target {
 }
 
 // NewHTTPClient returns a new HTTP client configured for the given scrape configuration.
-func NewHTTPClient(cfg *config.ScrapeConfig) (*http.Client, error) {
+func NewHTTPClient(cfg config.HTTPClientConfig) (*http.Client, error) {
 	tlsConfig, err := httputil.NewTLSConfig(cfg.TLSConfig)
 	if err != nil {
 		return nil, err

retrieval/target_test.go

@@ -151,9 +151,8 @@ func TestNewHTTPBearerToken(t *testing.T) {
 	)
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
+		BearerToken: "1234",
-		BearerToken:   "1234",
 	}
 	c, err := NewHTTPClient(cfg)
 	if err != nil {
@@ -179,8 +178,7 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
 	)
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout:   model.Duration(1 * time.Second),
 		BearerTokenFile: "testdata/bearertoken.txt",
 	}
 	c, err := NewHTTPClient(cfg)
@@ -206,8 +204,7 @@ func TestNewHTTPBasicAuth(t *testing.T) {
 	)
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		BasicAuth: &config.BasicAuth{
 			Username: "user",
 			Password: "password123",
@@ -236,8 +233,7 @@ func TestNewHTTPCACert(t *testing.T) {
 	server.StartTLS()
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		TLSConfig: config.TLSConfig{
 			CAFile: caCertPath,
 		},
@@ -269,8 +265,7 @@ func TestNewHTTPClientCert(t *testing.T) {
 	server.StartTLS()
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		TLSConfig: config.TLSConfig{
 			CAFile:   caCertPath,
 			CertFile: "testdata/client.cer",
@@ -300,8 +295,7 @@ func TestNewHTTPWithServerName(t *testing.T) {
 	server.StartTLS()
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		TLSConfig: config.TLSConfig{
 			CAFile:     caCertPath,
 			ServerName: "prometheus.rocks",
@@ -330,8 +324,7 @@ func TestNewHTTPWithBadServerName(t *testing.T) {
 	server.StartTLS()
 	defer server.Close()
 
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		TLSConfig: config.TLSConfig{
 			CAFile:     caCertPath,
 			ServerName: "badname",
@@ -369,8 +362,7 @@ func newTLSConfig(certName string, t *testing.T) *tls.Config {
 }
 
 func TestNewClientWithBadTLSConfig(t *testing.T) {
-	cfg := &config.ScrapeConfig{
+	cfg := config.HTTPClientConfig{
-		ScrapeTimeout: model.Duration(1 * time.Second),
 		TLSConfig: config.TLSConfig{
 			CAFile:   "testdata/nonexistent_ca.cer",
 			CertFile: "testdata/nonexistent_client.cer",

retrieval/targetmanager.go

@@ -120,7 +120,7 @@ func (tm *TargetManager) reload() {
 		} else {
 			ts.sp.reload(scfg)
 		}
-		ts.ts.UpdateProviders(discovery.ProvidersFromConfig(scfg))
+		ts.ts.UpdateProviders(discovery.ProvidersFromConfig(scfg.ServiceDiscoveryConfig))
 	}
 
 	// Remove old target sets. Waiting for scrape pools to complete pending

web/ui/bindata.go

@@ -120,7 +120,7 @@ func webUiTemplates_baseHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/_base.html", size: 2627, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/_base.html", size: 2627, mode: os.FileMode(420), modTime: time.Unix(1466006604, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -140,7 +140,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1795, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1795, mode: os.FileMode(420), modTime: time.Unix(1473241250, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -160,7 +160,7 @@ func webUiTemplatesConfigHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/config.html", size: 175, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/config.html", size: 175, mode: os.FileMode(420), modTime: time.Unix(1464223038, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -180,7 +180,7 @@ func webUiTemplatesFlagsHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/flags.html", size: 433, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/flags.html", size: 433, mode: os.FileMode(420), modTime: time.Unix(1464223038, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -200,7 +200,7 @@ func webUiTemplatesGraphHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/graph.html", size: 1645, mode: os.FileMode(420), modTime: time.Unix(1479349992, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/graph.html", size: 1645, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -220,12 +220,12 @@ func webUiTemplatesRulesHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/rules.html", size: 209, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/rules.html", size: 209, mode: os.FileMode(420), modTime: time.Unix(1464223038, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
 
-var _webUiTemplatesStatusHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xcc\x94\x31\x6f\x83\x30\x10\x85\x77\x7e\x85\xeb\x9d\x20\x65\x76\x3c\xa4\x95\xaa\xae\x51\xd3\x1d\xb8\x8b\x7c\x12\xb1\xa3\xb3\x43\x5b\x21\xff\xf7\xca\x04\x42\x22\xb5\x4d\x2a\x96\x2e\xe0\xc7\x3d\xdd\x77\x3c\x8c\xbb\x0e\x70\x47\x16\x85\x34\x58\x82\x8c\x51\x3d\xe4\xb9\xb0\xf4\x21\xf2\x5c\x77\x1d\x5a\x88\x31\xcb\x26\x57\xed\x6c\x40\x1b\x64\x8c\x99\x10\x0a\xa8\x15\x75\x53\x7a\xbf\xea\x0b\x25\x59\xe4\x7c\xd7\x1c\x09\xa4\xce\x84\x10\x42\x99\xa5\x20\x58\x49\x3e\xda\x40\x7b\x94\x7a\x73\x5a\x88\x17\xbb\x73\xbc\x2f\x03\x39\xab\x0a\xb3\x1c\xdc\xa1\xac\x1a\x1c\x3b\x9e\x44\x7f\xcd\x6b\x67\x01\xad\x47\x18\x74\xe5\x18\x90\xcf\xd2\x07\xa6\xc3\x59\x19\xd7\x22\x0f\x03\xa4\xa6\x95\x83\xcf\x51\x25\xcd\x93\x48\xd2\xe8\xed\x21\xcd\xa4\x8a\x60\xae\x2b\xa0\xbb\x6e\xb1\x26\x0e\x66\xb1\x7d\x7d\x8c\x51\x15\x01\x2e\x1a\x15\x53\x27\x55\x5c\x50\x54\xd1\xcf\xa1\xb3\xab\x08\xaa\x23\x35\x40\xd3\x6b\x4b\xbd\x4e\x4f\xfe\x55\x12\xc2\xd7\xee\x80\x2b\xc9\xee\x5d\xea\x37\x64\xdf\x0f\xf5\x6d\x2c\x43\x75\xbc\xff\x16\xce\x4d\xd2\x06\x5b\xba\x03\x35\xda\x66\xb1\xd6\x5c\xda\xda\xdc\x20\x9d\x4c\xf3\x38\xe9\xe3\x6e\x3d\xf2\x2d\xd4\xe8\x9b\x4f\x7b\x2a\xc3\x4f\x9b\xf8\x8a\x96\x7c\xb3\x68\xcf\xee\xbe\xbd\x71\xf6\xfd\xfd\xd7\x49\x4b\xa0\x56\x67\xe3\x19\xf4\x15\x00\x00\xff\xff\xed\x7c\x6c\xc8\xa9\x04\x00\x00")
+var _webUiTemplatesStatusHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xcc\x54\xb1\x0e\x9b\x30\x10\xdd\xf9\x0a\xd7\x3b\x41\xca\xec\x58\x6a\xda\xaa\xea\x1a\x35\xdd\x0d\xbe\xc4\x96\xc8\x19\x9d\x1d\xda\xca\xf2\xbf\x57\x10\x08\x41\x6a\x42\x24\x86\x76\x01\x1e\xf7\x74\xef\xf9\xd9\xbe\x18\x35\x9c\x2c\x02\xe3\x06\x94\xe6\x29\x89\x0f\x79\xce\xd0\xfe\x62\x79\x2e\x63\x04\xd4\x29\x65\xd9\xc4\xaa\x1c\x06\xc0\xc0\x53\xca\x18\x13\xda\xb6\xac\xaa\x95\xf7\xbb\xbe\xa0\x2c\x02\xe5\xa7\xfa\x6a\x35\x97\x19\x63\x8c\x09\xb3\x65\x56\xef\x38\x5d\x31\xd8\x0b\x70\x79\xb8\x7d\xb0\x6f\x78\x72\x74\x51\xc1\x3a\x14\x85\xd9\x0e\xec\xa0\xca\x1a\xc6\x8e\x37\xd0\x3f\xf3\xca\xa1\x06\xf4\xa0\x07\x5c\x3a\xd2\x40\x77\xe8\x03\xd9\xe6\x8e\x8c\x6b\x81\x06\x03\x5d\xd3\xd2\xe9\xdf\x23\xea\x30\x4d\xa0\x83\x46\x1e\x9b\xce\x93\x28\x82\x99\x57\xb4\x8c\x71\xb3\xb7\x14\xcc\xe6\xf8\xfd\x53\x4a\xa2\x08\xfa\xa1\x51\x31\x75\x12\xc5\x83\x8a\x28\x7a\x1f\x32\x9b\x45\x50\x5e\x6d\xad\xed\xb4\x6c\x2e\xf7\xdd\x9f\xff\x2a\x09\xe6\x2b\xd7\xc0\x8e\x93\xfb\xc9\xe5\x0f\x20\xdf\x9b\xfa\x6b\x2c\x43\x75\x7c\xbf\x0a\x67\x51\xe9\x00\xad\x7d\x43\x6a\xa4\xad\xd2\xda\x93\xc2\xca\x2c\x28\xdd\x48\xeb\x74\xba\xcd\x3d\x7a\xa0\x25\xa9\x91\xb7\x5e\xed\xb3\x0a\xcf\x0e\xf1\x4c\xad\xe3\xad\x52\xfb\xea\xde\x3b\x1b\x77\xde\xca\xab\xa3\x6a\xa0\x70\x51\xa8\xce\x40\x9e\xcb\x8f\x8f\xf0\xdf\xde\x99\x7e\x7c\x7c\x41\xdd\x38\x8b\x61\x9e\xc6\x3c\xd1\x18\x49\xe1\x19\xd8\x66\x66\xbe\x9f\xa2\x4f\x22\xef\x93\x7c\xbd\x4d\xe3\x7c\x5e\x08\x52\x14\xda\xb6\x32\x1b\xd9\x7f\x02\x00\x00\xff\xff\xcd\xdd\xd2\x78\xf3\x05\x00\x00")
 
 func webUiTemplatesStatusHtmlBytes() ([]byte, error) {
 	return bindataRead(
@@ -240,12 +240,12 @@ func webUiTemplatesStatusHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/status.html", size: 1193, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/status.html", size: 1523, mode: os.FileMode(420), modTime: time.Unix(1479996235, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
 
-var _webUiTemplatesTargetsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xbc\x55\xcd\x8e\xdb\x36\x10\xbe\xfb\x29\xa6\xec\xa2\xa7\xc8\x02\x02\xf4\xb2\xa5\x74\x68\x11\x20\x05\xb6\x45\x9a\x4d\x2e\xbd\x04\x94\x38\x96\xb8\xe1\x92\x2a\x39\x32\xb2\x60\xf8\xee\x05\x29\xc9\xb1\x65\xbb\x48\xba\x40\x7c\xa0\x35\xff\x7f\xdf\x90\x21\x48\xdc\x29\x83\xc0\x7a\x14\x92\xc5\xc8\x7f\x28\x0a\x30\xea\x13\x14\x45\x1d\x02\x1a\x19\xe3\x66\xf3\x45\xab\xb5\x86\xd0\x10\x8b\x71\x03\xc0\xa5\xda\x43\xab\x85\xf7\x55\x16\x08\x65\xd0\x15\x3b\x3d\x2a\xc9\xea\x0d\x00\x00\xef\x5f\x82\x92\x15\x23\xe1\x3a\x24\xcf\xea\x77\xd3\x07\x2f\xfb\x97\x93\x06\x00\x27\xd1\x68\x5c\xfc\x4c\x44\x3e\x8b\xd6\x1a\x89\xc6\xa3\x9c\xe9\xc6\x3a\x89\xee\x40\x7a\x72\x6a\x38\x50\xbd\xdd\xa3\x63\x8b\x53\x80\x10\x9c\x30\x1d\xc2\xcd\x83\x6d\x5e\xc0\xcd\x60\xad\x86\xdb\x0a\xb6\x6f\xac\xd5\x3e\xa7\xbf\xfc\x38\xa5\xda\xeb\x23\x4e\xe2\xb9\x9a\x53\x0f\xad\xd5\x7e\x10\xa6\x62\x3f\xb3\x25\xc3\x07\xdb\x7c\x48\x06\x29\x1a\x17\xb9\xbc\x07\xdb\x14\x21\xa4\x48\x31\x32\xe8\x1d\xee\x2a\xf6\xe3\x09\xb3\x5e\xbe\x78\x29\x6a\x5e\x52\x9f\x0e\x77\x1e\xf3\x84\x91\x53\xab\x5f\x19\x39\x58\x65\x28\x5b\x5d\x90\xdf\x93\x20\xbc\x26\xbc\x13\x0d\x6a\x7f\x5d\xea\x09\xee\x5b\x27\x86\xab\x0e\x5e\x39\x67\xdd\xb9\x70\x9d\x7d\xd2\x58\x35\x91\x53\x63\xe5\xd3\x31\xe7\x30\x92\x34\x8c\x93\x11\x5c\x29\x5e\x9e\xb1\xc4\xdc\xdd\x10\xb6\xef\xdf\xde\xc1\x67\xe8\xb4\x6d\x84\x7e\xff\xf6\x6e\x6a\x72\xe2\x6e\xef\xdb\x1e\x1f\x31\xc6\xdb\xb2\x9c\x39\xaf\xad\xa7\x18\x67\xe2\x8d\xa0\x7e\x1e\x44\x73\x16\xf4\x28\x4b\x9d\x7a\xf7\x02\x6e\xf6\x42\x8f\xe8\x33\x78\x92\xf9\x5f\x23\xba\x27\x58\xa5\xbf\x32\x55\x8b\x59\xb2\x9a\x1d\x5c\xb4\x00\xe0\x09\x5f\x0b\xb6\x72\x48\xc8\x67\x31\x38\xf5\x28\xdc\x53\x86\x4e\xe6\xc4\x98\xea\x9e\xbc\xc5\xc8\x78\x99\x2c\xcf\xf3\x4f\x69\x4c\x7b\xfb\x75\x7c\x5e\x5e\xe8\xf3\x39\x6b\x95\xa9\xd0\xe8\x08\xf2\x59\x84\x00\xdb\xd7\x28\x34\xf5\xf0\x19\xfa\xfc\xf1\xce\xfe\x96\xf4\x20\x46\xf0\x09\x9f\x1f\x94\x91\xaa\x15\x64\x1d\x10\x7e\xa2\x62\x1c\x06\x74\xad\xf0\xc8\x2e\x17\x30\xfb\xbb\x50\xc4\xe5\xb2\xff\x5f\x11\xed\xe8\xbc\x75\x45\x5e\x2f\x74\x0c\xa4\x20\x51\x90\xed\x3a\x8d\x15\x23\x6b\x35\xa9\x81\x01\x29\x4a\xf4\x2c\xee\xe9\x51\x57\xe4\x46\x9c\x48\xeb\x54\xa7\x8c\xd0\xc5\xac\xc5\x9b\xfa\x57\xdc\x59\x87\xe0\x30\x4f\x4d\x99\xee\x96\x97\x4d\x7d\xc0\xc6\xc7\x84\x8d\x8c\xa6\x3f\x90\xc4\xb4\xa0\x31\x26\x28\x86\x70\xf3\x31\x75\x90\x1e\xf5\xfc\x17\x63\xf5\xd3\x3f\xa3\xa5\x5f\xd2\xdc\xd7\xa2\x45\x92\x87\x7a\xa5\x8f\x13\x72\x32\x78\xf3\x75\x39\x85\x83\xed\xfc\x9f\x6e\x2e\xf6\xdf\x50\x3e\xd9\x82\x0c\x67\x3d\xa7\xfc\x1d\xe1\xac\x3d\x7e\x6b\x3c\x89\x3b\x31\x6a\x62\xb5\xb1\x06\xbf\x7d\x57\x9e\x09\xb3\x10\xd4\x2e\x75\xd9\xd3\x74\xbd\x6e\x7f\xf7\x7f\xa3\xb3\x31\xfe\x89\x7b\x74\x4b\x45\x21\x78\x65\x5a\x3c\x56\x8c\x11\x44\x67\x9f\xb9\xa9\x5f\xa2\xe7\xeb\xfb\x52\x79\xd7\x76\x59\xa6\xa9\xbb\xf5\xd2\xe6\xab\xf5\xc8\xdf\xb5\x7e\x7e\x6d\xde\xeb\xe7\xe3\xdc\x8e\x97\xab\xe7\xe3\x54\x85\x97\xf9\xd5\x4f\x62\x5e\x4a\xb5\xaf\x37\x8b\xfc\xdf\x00\x00\x00\xff\xff\x19\xaf\xc8\xb6\xd2\x08\x00\x00")
+var _webUiTemplatesTargetsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xbc\x56\xcd\x8e\xdb\x36\x10\xbe\xfb\x29\xa6\xec\xa2\xa7\xc8\x02\x02\xf4\xb2\xa5\x74\x68\x11\x20\x05\x36\x45\x9a\x4d\x2e\xbd\x04\x94\x38\x96\xb8\xe1\x92\x2a\x39\x32\xb2\x60\xf8\xee\x05\x29\xc9\xb1\x65\xbb\x48\xba\x40\x7d\xa0\x34\xff\x7f\xdf\x50\x0e\x41\xe2\x4e\x19\x04\xd6\xa3\x90\x2c\x46\xfe\x43\x51\x80\x51\x9f\xa1\x28\xea\x10\xd0\xc8\x18\x37\x9b\xaf\x5a\xad\x35\x84\x86\x58\x8c\x1b\x00\x2e\xd5\x1e\x5a\x2d\xbc\xaf\xb2\x40\x28\x83\xae\xd8\xe9\x51\x49\x56\x6f\x00\x00\x78\xff\x12\x94\xac\x18\x09\xd7\x21\x79\x56\xbf\x9f\x5e\x78\xd9\xbf\x9c\x34\x00\x38\x89\x46\xe3\xe2\x67\x22\xf2\x59\xb4\xd6\x48\x34\x1e\xe5\x4c\x37\xd6\x49\x74\x07\xd2\x93\x53\xc3\x81\xea\xed\x1e\x1d\x5b\x9c\x02\x84\xe0\x84\xe9\x10\x6e\x1e\x6c\xf3\x02\x6e\x06\x6b\x35\xdc\x56\xb0\x9d\x32\x78\x23\x8c\xe8\xd0\x6d\xdf\x5a\xab\x7d\x2e\x66\xf9\x71\x4a\x9d\xa8\x8f\x38\x89\xe7\x6a\x4e\x3d\xb4\x56\xfb\x41\x98\x8a\xfd\xcc\x96\x7c\x1f\x6c\xf3\x31\x19\xa4\xd8\x5c\xe4\x62\x1f\x6c\x53\x84\x90\xe2\xc6\xc8\xa0\x77\xb8\xab\xd8\x8f\x27\xcc\x7a\x79\xe3\xa5\xa8\x79\x49\x7d\x3a\xdc\x79\xcc\x13\x46\x4e\xad\x7e\x65\xe4\x60\x95\xa1\x6c\x75\x41\x7e\x4f\x82\xf0\x9a\xf0\x4e\x34\xa8\xfd\x75\xa9\x27\xb8\x6f\x9d\x18\xae\x3a\x78\xe5\x9c\x75\xe7\xc2\x75\xf6\x49\x63\xd5\x44\x4e\x8d\x95\x4f\xc7\x9c\xc3\x80\xd2\x68\x4e\x46\x70\xa5\x78\x79\xc6\x12\x73\x77\x43\xd8\x7e\x78\x77\x07\x5f\xa0\xd3\xb6\x11\xfa\xc3\xbb\xbb\xa9\xc9\x89\xbb\xbd\x6f\x7b\x7c\xc4\x18\x6f\xcb\x72\xe6\xbc\xb6\x9e\x62\x9c\x89\xb7\x82\xfa\x79\x10\xcd\x59\xd0\xa3\x2c\x75\xea\xdd\x0b\xb8\xd9\x0b\x3d\xa2\xcf\x50\x4a\xe6\x7f\x8e\xe8\x9e\x60\x95\xfe\xca\x54\x2d\x66\xc9\x6a\x76\x70\xd1\x02\x80\x27\x7c\x2d\xd8\xca\x21\x21\x9f\xc5\xe0\xd4\xa3\x70\x4f\x19\x3a\x99\x13\x63\xaa\x7b\xf2\x16\x23\xe3\x65\xb2\x3c\xcf\x3f\xa5\x31\x6d\xf1\xb7\xf1\x79\x79\xa1\xcf\xe7\xac\x55\xa6\x42\xa3\x23\xc8\x67\x11\x02\x6c\x5f\xa3\xd0\xd4\xc3\x17\xe8\xf3\xcb\x7b\xfb\x5b\xd2\x83\x18\xc1\x27\x7c\x7e\x54\x46\xaa\x56\x90\x75\x40\xf8\x99\x8a\x71\x18\xd0\xb5\xc2\x23\xbb\x5c\xc0\xec\xef\x42\x11\x97\xcb\xfe\x6f\x45\xb4\xa3\xf3\xd6\x15\x79\xbd\xd0\x31\x90\x82\x44\x41\xb6\xeb\x34\x56\x8c\xac\xd5\xa4\x06\x06\xa4\x28\xd1\xb3\xb8\xa7\x47\x5d\x91\x1b\x71\x22\xad\x53\x9d\x32\x42\x17\xb3\x16\x6f\xea\x5f\x71\x67\x1d\x82\xc3\x3c\x35\x65\xba\x5b\x5e\x36\xf5\x01\x1b\x9f\x12\x36\x32\x9a\xde\x20\x89\x69\x41\x63\x4c\x50\x0c\xe1\xe6\x53\xea\x20\x3d\xea\xf9\x11\x63\xf5\xd3\xdf\xa3\xa5\x5f\xd2\xdc\xd7\xa2\x45\x92\x87\x7a\xa5\x8f\x13\x72\x32\x78\xf3\xe5\x39\x85\x83\xed\xfc\x4c\x37\x17\xfb\x77\x28\x9f\x6c\x41\x86\xb3\x9e\x53\xfe\x1f\xe1\xac\x3d\x7e\x6f\x3c\x89\x3b\x31\x6a\x62\xb5\xb1\x06\xbf\x7f\x57\x9e\x09\xb3\x10\xd4\x2e\x75\xd9\xd3\x74\xbd\x6e\x7f\xf7\x7f\xa1\xb3\x31\xfe\x81\x7b\x74\x4b\x45\x21\x78\x65\x5a\x3c\x56\x8c\x11\x44\x67\x9f\xb9\xa9\x5f\xa3\xe7\xeb\xfb\x52\x79\xd7\x76\x59\xa6\xa9\xbb\xf5\xd2\xe6\xab\xf5\xc8\xdf\xb5\x7e\x7e\x6b\xde\xeb\xcf\xc7\xb9\x1d\x2f\x57\x9f\x8f\x53\x15\x5e\xe6\xff\x00\x49\xcc\x4b\xa9\xf6\xf5\x66\x91\xff\x13\x00\x00\xff\xff\xef\x54\xd5\x0a\xe0\x08\x00\x00")
 
 func webUiTemplatesTargetsHtmlBytes() ([]byte, error) {
 	return bindataRead(
@@ -260,7 +260,7 @@ func webUiTemplatesTargetsHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/targets.html", size: 2258, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/targets.html", size: 2272, mode: os.FileMode(420), modTime: time.Unix(1479996224, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -280,7 +280,7 @@ func webUiStaticCssAlertsCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/css/alerts.css", size: 74, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/css/alerts.css", size: 74, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -300,7 +300,7 @@ func webUiStaticCssGraphCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/css/graph.css", size: 2668, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/css/graph.css", size: 2668, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -320,7 +320,7 @@ func webUiStaticCssProm_consoleCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/css/prom_console.css", size: 2883, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/css/prom_console.css", size: 2883, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -340,7 +340,7 @@ func webUiStaticCssPrometheusCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/css/prometheus.css", size: 405, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/static/css/prometheus.css", size: 405, mode: os.FileMode(420), modTime: time.Unix(1462274250, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -360,7 +360,7 @@ func webUiStaticImgAjaxLoaderGif() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -380,7 +380,7 @@ func webUiStaticJsAlertsJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/js/alerts.js", size: 445, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/js/alerts.js", size: 445, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -400,7 +400,7 @@ func webUiStaticJsGraphJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/js/graph.js", size: 25570, mode: os.FileMode(420), modTime: time.Unix(1479349992, 0)}
+	info := bindataFileInfo{name: "web/ui/static/js/graph.js", size: 25570, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -420,7 +420,7 @@ func webUiStaticJsGraph_templateHandlebar() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/js/graph_template.handlebar", size: 6316, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/static/js/graph_template.handlebar", size: 6316, mode: os.FileMode(420), modTime: time.Unix(1473241250, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -440,7 +440,7 @@ func webUiStaticJsProm_consoleJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/js/prom_console.js", size: 21219, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/static/js/prom_console.js", size: 21219, mode: os.FileMode(420), modTime: time.Unix(1473241250, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -460,7 +460,7 @@ func webUiStaticVendorBootstrap331CssBootstrapThemeMinCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css", size: 19835, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css", size: 19835, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -480,7 +480,7 @@ func webUiStaticVendorBootstrap331CssBootstrapMinCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css", size: 113498, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css", size: 113498, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -500,7 +500,7 @@ func webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot() (*asset,
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot", size: 20335, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot", size: 20335, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -520,7 +520,7 @@ func webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg() (*asset,
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg", size: 62926, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg", size: 62926, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -540,7 +540,7 @@ func webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf() (*asset,
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf", size: 41280, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf", size: 41280, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -560,7 +560,7 @@ func webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff() (*asset,
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff", size: 23320, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff", size: 23320, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -580,7 +580,7 @@ func webUiStaticVendorBootstrap331JsBootstrapMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js", size: 35601, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js", size: 35601, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -600,7 +600,7 @@ func webUiStaticVendorBootstrap331JsNpmJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js", size: 484, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js", size: 484, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -620,7 +620,7 @@ func webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs() (*asset, err
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 7856, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 7856, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -640,7 +640,7 @@ func webUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinC
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1479345773, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -660,7 +660,7 @@ func webUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinJ
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1479345773, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -680,7 +680,7 @@ func webUiStaticVendorFuzzyJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/fuzzy.js", size: 2655, mode: os.FileMode(420), modTime: time.Unix(1479339266, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/fuzzy.js", size: 2655, mode: os.FileMode(420), modTime: time.Unix(1479736746, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -700,7 +700,7 @@ func webUiStaticVendorJsJqueryHotkeysJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.hotkeys.js", size: 3283, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.hotkeys.js", size: 3283, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -720,7 +720,7 @@ func webUiStaticVendorJsJqueryMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.min.js", size: 95935, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.min.js", size: 95935, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -740,7 +740,7 @@ func webUiStaticVendorJsJquerySelectionJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.selection.js", size: 13320, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/js/jquery.selection.js", size: 13320, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -760,7 +760,7 @@ func webUiStaticVendorMomentMomentMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/moment/moment.min.js", size: 61281, mode: os.FileMode(420), modTime: time.Unix(1479345773, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/moment/moment.min.js", size: 61281, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -780,7 +780,7 @@ func webUiStaticVendorMustacheMustacheMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1479349992, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1479736906, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -800,7 +800,7 @@ func webUiStaticVendorRickshawRickshawMinCss() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -820,7 +820,7 @@ func webUiStaticVendorRickshawRickshawMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -840,7 +840,7 @@ func webUiStaticVendorRickshawVendorD3LayoutMinJs() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -860,7 +860,7 @@ func webUiStaticVendorRickshawVendorD3V3Js() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1479339108, 0)}
+	info := bindataFileInfo{name: "web/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1461244866, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }

web/ui/templates/status.html

@@ -41,5 +41,20 @@
         </tr>
       </tbody>
     </table>
+
+    <h2 id="alertmanagers">Alertmanagers</h2>
+    <table class="table table-condensed table-bordered table-striped table-hover">
+      <tbody>
+        <tr>
+           <th>Endpoint</th>
+        </tr>
+        {{range .Alertmanagers}}
+        <tr>
+          <td>{{.}}</td>
+        </tr>
+        {{end}}
+      </tbody>
+    </table>
+
   </div>
 {{end}}

web/ui/templates/targets.html

@@ -4,7 +4,7 @@
   <div class="container-fluid">
     <h2 id="targets">Targets</h2>
       <table class="table table-condensed table-bordered table-striped table-hover">
-        {{range $job, $pool := .Pools}}
+        {{range $job, $pool := .TargetManager.Pools}}
           <thead>
             <tr><th colspan="5" class="job_header"><a id="job-{{$job}}" href="#job-{{$job}}">{{$job}}</a></th></tr>
             <tr>

web/web.go

@@ -39,6 +39,7 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/retrieval"
 	"github.com/prometheus/prometheus/rules"
@@ -58,6 +59,7 @@ type Handler struct {
 	queryEngine   *promql.Engine
 	context       context.Context
 	storage       local.Storage
+	notifier      *notifier.Notifier
 
 	apiV1 *api_v1.API
 
@@ -104,6 +106,7 @@ type Options struct {
 	QueryEngine   *promql.Engine
 	TargetManager *retrieval.TargetManager
 	RuleManager   *rules.Manager
+	Notifier      *notifier.Notifier
 	Version       *PrometheusVersion
 	Flags         map[string]string
 
@@ -139,6 +142,7 @@ func New(o *Options) *Handler {
 		ruleManager:   o.RuleManager,
 		queryEngine:   o.QueryEngine,
 		storage:       o.Storage,
+		notifier:      o.Notifier,
 
 		apiV1: api_v1.NewAPI(o.QueryEngine, o.Storage),
 		now:   model.Now,
@@ -322,11 +326,13 @@ func (h *Handler) graph(w http.ResponseWriter, r *http.Request) {
 
 func (h *Handler) status(w http.ResponseWriter, r *http.Request) {
 	h.executeTemplate(w, "status.html", struct {
-		Birth   time.Time
+		Birth         time.Time
-		Version *PrometheusVersion
+		Version       *PrometheusVersion
+		Alertmanagers []string
 	}{
-		Birth:   h.birth,
+		Birth:         h.birth,
-		Version: h.versionInfo,
+		Version:       h.versionInfo,
+		Alertmanagers: h.notifier.Alertmanagers(),
 	})
 }
 
@@ -346,7 +352,11 @@ func (h *Handler) rules(w http.ResponseWriter, r *http.Request) {
 }
 
 func (h *Handler) targets(w http.ResponseWriter, r *http.Request) {
-	h.executeTemplate(w, "targets.html", h.targetManager)
+	h.executeTemplate(w, "targets.html", struct {
+		TargetManager *retrieval.TargetManager
+	}{
+		TargetManager: h.targetManager,
+	})
 }
 
 func (h *Handler) version(w http.ResponseWriter, r *http.Request) {