diff --git a/config/config_test.go b/config/config_test.go index 330f6f5a44..de6ad85264 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -30,11 +30,11 @@ import ( "gopkg.in/yaml.v2" "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/aws" "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" "github.com/prometheus/prometheus/discovery/digitalocean" "github.com/prometheus/prometheus/discovery/dns" - "github.com/prometheus/prometheus/discovery/ec2" "github.com/prometheus/prometheus/discovery/eureka" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/hetzner" @@ -465,14 +465,14 @@ var expectedConf = &Config{ HTTPClientConfig: config.DefaultHTTPClientConfig, ServiceDiscoveryConfigs: discovery.Configs{ - &ec2.SDConfig{ + &aws.EC2SDConfig{ Region: "us-east-1", AccessKey: "access", SecretKey: "mysecret", Profile: "profile", RefreshInterval: model.Duration(60 * time.Second), Port: 80, - Filters: []*ec2.Filter{ + Filters: []*aws.EC2Filter{ { Name: "tag:environment", Values: []string{"prod"}, @@ -485,6 +485,28 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "service-lightsail", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + + ServiceDiscoveryConfigs: discovery.Configs{ + &aws.LightsailSDConfig{ + Region: "us-east-1", + AccessKey: "access", + SecretKey: "mysecret", + Profile: "profile", + RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + }, + }, + }, { JobName: "service-azure", @@ -886,7 +908,7 @@ func TestElideSecrets(t *testing.T) { yamlConfig := string(config) matches := secretRe.FindAllStringIndex(yamlConfig, -1) - require.Equal(t, 12, len(matches), "wrong number of secret matches found") + require.Equal(t, 13, len(matches), "wrong number of secret matches found") require.NotContains(t, yamlConfig, "mysecret", "yaml marshal reveals authentication credentials.") } diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 515f91ed9b..7163abbfe1 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -215,6 +215,13 @@ scrape_configs: - web - db +- job_name: service-lightsail + lightsail_sd_configs: + - region: us-east-1 + access_key: access + secret_key: mysecret + profile: profile + - job_name: service-azure azure_sd_configs: - environment: AzurePublicCloud diff --git a/discovery/ec2/ec2.go b/discovery/aws/ec2.go similarity index 85% rename from discovery/ec2/ec2.go rename to discovery/aws/ec2.go index 8fa694c86b..e2464277ab 100644 --- a/discovery/ec2/ec2.go +++ b/discovery/aws/ec2.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright 2021 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ec2 +package aws import ( "context" @@ -61,24 +61,26 @@ const ( ec2LabelSeparator = "," ) -// DefaultSDConfig is the default EC2 SD configuration. -var DefaultSDConfig = SDConfig{ - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), -} +var ( + // DefaultEC2SDConfig is the default EC2 SD configuration. + DefaultEC2SDConfig = EC2SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + } +) func init() { - discovery.RegisterConfig(&SDConfig{}) + discovery.RegisterConfig(&EC2SDConfig{}) } // Filter is the configuration for filtering EC2 instances. -type Filter struct { +type EC2Filter struct { Name string `yaml:"name"` Values []string `yaml:"values"` } -// SDConfig is the configuration for EC2 based service discovery. -type SDConfig struct { +// EC2SDConfig is the configuration for EC2 based service discovery. +type EC2SDConfig struct { Endpoint string `yaml:"endpoint"` Region string `yaml:"region"` AccessKey string `yaml:"access_key,omitempty"` @@ -87,21 +89,21 @@ type SDConfig struct { RoleARN string `yaml:"role_arn,omitempty"` RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` Port int `yaml:"port"` - Filters []*Filter `yaml:"filters"` + Filters []*EC2Filter `yaml:"filters"` } -// Name returns the name of the Config. -func (*SDConfig) Name() string { return "ec2" } +// Name returns the name of the EC2 Config. +func (*EC2SDConfig) Name() string { return "ec2" } -// NewDiscoverer returns a Discoverer for the Config. -func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger), nil +// NewDiscoverer returns a Discoverer for the EC2 Config. +func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewEC2Discovery(c, opts.Logger), nil } -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultSDConfig - type plain SDConfig +// UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config. +func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultEC2SDConfig + type plain EC2SDConfig err := unmarshal((*plain)(c)) if err != nil { return err @@ -128,18 +130,18 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // Discovery periodically performs EC2-SD requests. It implements // the Discoverer interface. -type Discovery struct { +type EC2Discovery struct { *refresh.Discovery - cfg *SDConfig + cfg *EC2SDConfig ec2 *ec2.EC2 } -// NewDiscovery returns a new EC2Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery { +// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. +func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery { if logger == nil { logger = log.NewNopLogger() } - d := &Discovery{ + d := &EC2Discovery{ cfg: conf, } d.Discovery = refresh.NewDiscovery( @@ -151,7 +153,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery { return d } -func (d *Discovery) ec2Client() (*ec2.EC2, error) { +func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) { if d.ec2 != nil { return d.ec2, nil } @@ -183,7 +185,7 @@ func (d *Discovery) ec2Client() (*ec2.EC2, error) { return d.ec2, nil } -func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { +func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { ec2Client, err := d.ec2Client() if err != nil { return nil, err diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go new file mode 100644 index 0000000000..8e11a1a4d6 --- /dev/null +++ b/discovery/aws/lightsail.go @@ -0,0 +1,234 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "fmt" + "net" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/lightsail" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + lightsailLabel = model.MetaLabelPrefix + "lightsail_" + lightsailLabelAZ = lightsailLabel + "availability_zone" + lightsailLabelBlueprintID = lightsailLabel + "blueprint_id" + lightsailLabelBundleID = lightsailLabel + "bundle_id" + lightsailLabelInstanceName = lightsailLabel + "instance_name" + lightsailLabelInstanceState = lightsailLabel + "instance_state" + lightsailLabelInstanceSupportCode = lightsailLabel + "instance_support_code" + lightsailLabelIPv6Addresses = lightsailLabel + "ipv6_addresses" + lightsailLabelPrivateIP = lightsailLabel + "private_ip" + lightsailLabelPublicIP = lightsailLabel + "public_ip" + lightsailLabelTag = lightsailLabel + "tag_" + lightsailLabelSeparator = "," +) + +var ( + // DefaultLightsailSDConfig is the default Lightsail SD configuration. + DefaultLightsailSDConfig = LightsailSDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + } +) + +func init() { + discovery.RegisterConfig(&LightsailSDConfig{}) +} + +// LightsailSDConfig is the configuration for Lightsail based service discovery. +type LightsailSDConfig struct { + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` +} + +// Name returns the name of the Lightsail Config. +func (*LightsailSDConfig) Name() string { return "lightsail" } + +// NewDiscoverer returns a Discoverer for the Lightsail Config. +func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewLightsailDiscovery(c, opts.Logger), nil +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config. +func (c *LightsailSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultLightsailSDConfig + type plain LightsailSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.Region == "" { + sess, err := session.NewSession() + if err != nil { + return err + } + + metadata := ec2metadata.New(sess) + + region, err := metadata.Region() + if err != nil { + return errors.New("Lightsail SD configuration requires a region") + } + c.Region = region + } + return nil +} + +// Discovery periodically performs Lightsail-SD requests. It implements +// the Discoverer interface. +type LightsailDiscovery struct { + *refresh.Discovery + cfg *LightsailSDConfig + lightsail *lightsail.Lightsail +} + +// NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets. +func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery { + if logger == nil { + logger = log.NewNopLogger() + } + d := &LightsailDiscovery{ + cfg: conf, + } + d.Discovery = refresh.NewDiscovery( + logger, + "lightsail", + time.Duration(d.cfg.RefreshInterval), + d.refresh, + ) + return d +} + +func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) { + if d.lightsail != nil { + return d.lightsail, nil + } + + creds := credentials.NewStaticCredentials(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + if d.cfg.AccessKey == "" && d.cfg.SecretKey == "" { + creds = nil + } + + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Endpoint: &d.cfg.Endpoint, + Region: &d.cfg.Region, + Credentials: creds, + }, + Profile: d.cfg.Profile, + }) + if err != nil { + return nil, errors.Wrap(err, "could not create aws session") + } + + if d.cfg.RoleARN != "" { + creds := stscreds.NewCredentials(sess, d.cfg.RoleARN) + d.lightsail = lightsail.New(sess, &aws.Config{Credentials: creds}) + } else { + d.lightsail = lightsail.New(sess) + } + + return d.lightsail, nil +} + +func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + lightsailClient, err := d.lightsailClient() + if err != nil { + return nil, err + } + + tg := &targetgroup.Group{ + Source: d.cfg.Region, + } + + input := &lightsail.GetInstancesInput{} + + output, err := lightsailClient.GetInstancesWithContext(ctx, input) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && (awsErr.Code() == "AuthFailure" || awsErr.Code() == "UnauthorizedOperation") { + d.lightsail = nil + } + return nil, errors.Wrap(err, "could not get instances") + } + + for _, inst := range output.Instances { + if inst.PrivateIpAddress == nil { + continue + } + + labels := model.LabelSet{ + lightsailLabelAZ: model.LabelValue(*inst.Location.AvailabilityZone), + lightsailLabelBlueprintID: model.LabelValue(*inst.BlueprintId), + lightsailLabelBundleID: model.LabelValue(*inst.BundleId), + lightsailLabelInstanceName: model.LabelValue(*inst.Name), + lightsailLabelInstanceState: model.LabelValue(*inst.State.Name), + lightsailLabelInstanceSupportCode: model.LabelValue(*inst.SupportCode), + lightsailLabelPrivateIP: model.LabelValue(*inst.PrivateIpAddress), + } + + addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.cfg.Port)) + labels[model.AddressLabel] = model.LabelValue(addr) + + if inst.PublicIpAddress != nil { + labels[lightsailLabelPublicIP] = model.LabelValue(*inst.PublicIpAddress) + } + + if len(inst.Ipv6Addresses) > 0 { + var ipv6addrs []string + for _, ipv6addr := range inst.Ipv6Addresses { + ipv6addrs = append(ipv6addrs, *ipv6addr) + } + labels[lightsailLabelIPv6Addresses] = model.LabelValue( + lightsailLabelSeparator + + strings.Join(ipv6addrs, lightsailLabelSeparator) + + lightsailLabelSeparator) + } + + for _, t := range inst.Tags { + if t == nil || t.Key == nil || t.Value == nil { + continue + } + name := strutil.SanitizeLabelName(*t.Key) + labels[lightsailLabelTag+model.LabelName(name)] = model.LabelValue(*t.Value) + } + + tg.Targets = append(tg.Targets, labels) + } + return []*targetgroup.Group{tg}, nil +} diff --git a/discovery/install/install.go b/discovery/install/install.go index d6f69d0d66..3e6f0f388e 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -16,11 +16,11 @@ package install import ( + _ "github.com/prometheus/prometheus/discovery/aws" // register aws _ "github.com/prometheus/prometheus/discovery/azure" // register azure _ "github.com/prometheus/prometheus/discovery/consul" // register consul _ "github.com/prometheus/prometheus/discovery/digitalocean" // register digitalocean _ "github.com/prometheus/prometheus/discovery/dns" // register dns - _ "github.com/prometheus/prometheus/discovery/ec2" // register ec2 _ "github.com/prometheus/prometheus/discovery/eureka" // register eureka _ "github.com/prometheus/prometheus/discovery/file" // register file _ "github.com/prometheus/prometheus/discovery/gce" // register gce diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 7d9a0e9b29..d2bc058cd3 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -238,6 +238,10 @@ hetzner_sd_configs: kubernetes_sd_configs: [ - ... ] +# List of Lightsail service discovery configurations. +lightsail_sd_configs: + [ - ... ] + # List of Marathon service discovery configurations. marathon_sd_configs: [ - ... ] @@ -1341,6 +1345,54 @@ for a detailed example of configuring Prometheus for Kubernetes. You may wish to check out the 3rd party [Prometheus Operator](https://github.com/coreos/prometheus-operator), which automates the Prometheus setup on top of Kubernetes. +### `` + +Lightsail SD configurations allow retrieving scrape targets from [AWS Lightsail](https://aws.amazon.com/lightsail/) +instances. The private IP address is used by default, but may be changed to +the public IP address with relabeling. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_lightsail_availability_zone`: the availability zone in which the instance is running +* `__meta_lightsail_blueprint_id`: the Lightsail blueprint ID +* `__meta_lightsail_bundle_id`: the Lightsail bundle ID +* `__meta_lightsail_instance_name`: the name of the Lightsail instance +* `__meta_lightsail_instance_state`: the state of the Lightsail instance +* `__meta_lightsail_instance_support_code`: the support code of the Lightsail instance +* `__meta_lightsail_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present +* `__meta_lightsail_private_ip`: the private IP address of the instance +* `__meta_lightsail_public_ip`: the public IP address of the instance, if available +* `__meta_lightsail_tag_`: each tag value of the instance + +See below for the configuration options for Lightsail discovery: + +```yaml +# The information to access the Lightsail API. + +# The AWS region. If blank, the region from the instance metadata is used. +[ region: ] + +# Custom endpoint to be used. +[ endpoint: ] + +# The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID` +# and `AWS_SECRET_ACCESS_KEY` are used. +[ access_key: ] +[ secret_key: ] +# Named AWS profile used to connect to the API. +[ profile: ] + +# AWS Role ARN, an alternative to using AWS API keys. +[ role_arn: ] + +# Refresh interval to re-read the instance list. +[ refresh_interval: | default = 60s ] + +# The port to scrape metrics from. If using the public IP address, this must +# instead be specified in the relabeling rule. +[ port: | default = 80 ] +``` + ### `` Marathon SD configurations allow retrieving scrape targets using the @@ -1936,6 +1988,10 @@ hetzner_sd_configs: kubernetes_sd_configs: [ - ... ] +# List of Lightsail service discovery configurations. +lightsail_sd_configs: + [ - ... ] + # List of Marathon service discovery configurations. marathon_sd_configs: [ - ... ]