Add lightsail service discovery (#8693)

Signed-off-by: N888 <drifto@gmail.com>

config/config_test.go

@@ -30,11 +30,11 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/aws"
 	"github.com/prometheus/prometheus/discovery/azure"
 	"github.com/prometheus/prometheus/discovery/consul"
 	"github.com/prometheus/prometheus/discovery/digitalocean"
 	"github.com/prometheus/prometheus/discovery/dns"
-	"github.com/prometheus/prometheus/discovery/ec2"
 	"github.com/prometheus/prometheus/discovery/eureka"
 	"github.com/prometheus/prometheus/discovery/file"
 	"github.com/prometheus/prometheus/discovery/hetzner"
@@ -465,14 +465,14 @@ var expectedConf = &Config{
 			HTTPClientConfig: config.DefaultHTTPClientConfig,
 
 			ServiceDiscoveryConfigs: discovery.Configs{
-				&ec2.SDConfig{
+				&aws.EC2SDConfig{
 					Region:          "us-east-1",
 					AccessKey:       "access",
 					SecretKey:       "mysecret",
 					Profile:         "profile",
 					RefreshInterval: model.Duration(60 * time.Second),
 					Port:            80,
-					Filters: []*ec2.Filter{
+					Filters: []*aws.EC2Filter{
 						{
 							Name:   "tag:environment",
 							Values: []string{"prod"},
@@ -485,6 +485,28 @@ var expectedConf = &Config{
 				},
 			},
 		},
+		{
+			JobName: "service-lightsail",
+
+			HonorTimestamps: true,
+			ScrapeInterval:  model.Duration(15 * time.Second),
+			ScrapeTimeout:   DefaultGlobalConfig.ScrapeTimeout,
+
+			MetricsPath:      DefaultScrapeConfig.MetricsPath,
+			Scheme:           DefaultScrapeConfig.Scheme,
+			HTTPClientConfig: config.DefaultHTTPClientConfig,
+
+			ServiceDiscoveryConfigs: discovery.Configs{
+				&aws.LightsailSDConfig{
+					Region:          "us-east-1",
+					AccessKey:       "access",
+					SecretKey:       "mysecret",
+					Profile:         "profile",
+					RefreshInterval: model.Duration(60 * time.Second),
+					Port:            80,
+				},
+			},
+		},
 		{
 			JobName: "service-azure",
 
@@ -886,7 +908,7 @@ func TestElideSecrets(t *testing.T) {
 	yamlConfig := string(config)
 
 	matches := secretRe.FindAllStringIndex(yamlConfig, -1)
-	require.Equal(t, 12, len(matches), "wrong number of secret matches found")
+	require.Equal(t, 13, len(matches), "wrong number of secret matches found")
 	require.NotContains(t, yamlConfig, "mysecret",
 		"yaml marshal reveals authentication credentials.")
 }

config/testdata/conf.good.yml

@@ -215,6 +215,13 @@ scrape_configs:
             - web
             - db
 
+- job_name: service-lightsail
+  lightsail_sd_configs:
+    - region: us-east-1
+      access_key: access
+      secret_key: mysecret
+      profile: profile
+
 - job_name: service-azure
   azure_sd_configs:
     - environment: AzurePublicCloud

discovery/aws/ec2.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Prometheus Authors
+// Copyright 2021 The Prometheus Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ec2
+package aws
 
 import (
 	"context"
@@ -61,24 +61,26 @@ const (
 	ec2LabelSeparator         = ","
 )
 
-// DefaultSDConfig is the default EC2 SD configuration.
+var (
-var DefaultSDConfig = SDConfig{
+	// DefaultEC2SDConfig is the default EC2 SD configuration.
-	Port:            80,
+	DefaultEC2SDConfig = EC2SDConfig{
-	RefreshInterval: model.Duration(60 * time.Second),
+		Port:            80,
-}
+		RefreshInterval: model.Duration(60 * time.Second),
+	}
+)
 
 func init() {
-	discovery.RegisterConfig(&SDConfig{})
+	discovery.RegisterConfig(&EC2SDConfig{})
 }
 
 // Filter is the configuration for filtering EC2 instances.
-type Filter struct {
+type EC2Filter struct {
 	Name   string   `yaml:"name"`
 	Values []string `yaml:"values"`
 }
 
-// SDConfig is the configuration for EC2 based service discovery.
+// EC2SDConfig is the configuration for EC2 based service discovery.
-type SDConfig struct {
+type EC2SDConfig struct {
 	Endpoint        string         `yaml:"endpoint"`
 	Region          string         `yaml:"region"`
 	AccessKey       string         `yaml:"access_key,omitempty"`
@@ -87,21 +89,21 @@ type SDConfig struct {
 	RoleARN         string         `yaml:"role_arn,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 	Port            int            `yaml:"port"`
-	Filters         []*Filter      `yaml:"filters"`
+	Filters         []*EC2Filter   `yaml:"filters"`
 }
 
-// Name returns the name of the Config.
+// Name returns the name of the EC2 Config.
-func (*SDConfig) Name() string { return "ec2" }
+func (*EC2SDConfig) Name() string { return "ec2" }
 
-// NewDiscoverer returns a Discoverer for the Config.
+// NewDiscoverer returns a Discoverer for the EC2 Config.
-func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewEC2Discovery(c, opts.Logger), nil
 }
 
-// UnmarshalYAML implements the yaml.Unmarshaler interface.
+// UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
-func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	*c = DefaultSDConfig
+	*c = DefaultEC2SDConfig
-	type plain SDConfig
+	type plain EC2SDConfig
 	err := unmarshal((*plain)(c))
 	if err != nil {
 		return err
@@ -128,18 +130,18 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 // Discovery periodically performs EC2-SD requests. It implements
 // the Discoverer interface.
-type Discovery struct {
+type EC2Discovery struct {
 	*refresh.Discovery
-	cfg *SDConfig
+	cfg *EC2SDConfig
 	ec2 *ec2.EC2
 }
 
-// NewDiscovery returns a new EC2Discovery which periodically refreshes its targets.
+// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
-	d := &Discovery{
+	d := &EC2Discovery{
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
@@ -151,7 +153,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
 	return d
 }
 
-func (d *Discovery) ec2Client() (*ec2.EC2, error) {
+func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) {
 	if d.ec2 != nil {
 		return d.ec2, nil
 	}
@@ -183,7 +185,7 @@ func (d *Discovery) ec2Client() (*ec2.EC2, error) {
 	return d.ec2, nil
 }
 
-func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	ec2Client, err := d.ec2Client()
 	if err != nil {
 		return nil, err

discovery/aws/lightsail.go

@@ -0,0 +1,234 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/lightsail"
+	"github.com/go-kit/kit/log"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/refresh"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"github.com/prometheus/prometheus/util/strutil"
+)
+
+const (
+	lightsailLabel                    = model.MetaLabelPrefix + "lightsail_"
+	lightsailLabelAZ                  = lightsailLabel + "availability_zone"
+	lightsailLabelBlueprintID         = lightsailLabel + "blueprint_id"
+	lightsailLabelBundleID            = lightsailLabel + "bundle_id"
+	lightsailLabelInstanceName        = lightsailLabel + "instance_name"
+	lightsailLabelInstanceState       = lightsailLabel + "instance_state"
+	lightsailLabelInstanceSupportCode = lightsailLabel + "instance_support_code"
+	lightsailLabelIPv6Addresses       = lightsailLabel + "ipv6_addresses"
+	lightsailLabelPrivateIP           = lightsailLabel + "private_ip"
+	lightsailLabelPublicIP            = lightsailLabel + "public_ip"
+	lightsailLabelTag                 = lightsailLabel + "tag_"
+	lightsailLabelSeparator           = ","
+)
+
+var (
+	// DefaultLightsailSDConfig is the default Lightsail SD configuration.
+	DefaultLightsailSDConfig = LightsailSDConfig{
+		Port:            80,
+		RefreshInterval: model.Duration(60 * time.Second),
+	}
+)
+
+func init() {
+	discovery.RegisterConfig(&LightsailSDConfig{})
+}
+
+// LightsailSDConfig is the configuration for Lightsail based service discovery.
+type LightsailSDConfig struct {
+	Endpoint        string         `yaml:"endpoint"`
+	Region          string         `yaml:"region"`
+	AccessKey       string         `yaml:"access_key,omitempty"`
+	SecretKey       config.Secret  `yaml:"secret_key,omitempty"`
+	Profile         string         `yaml:"profile,omitempty"`
+	RoleARN         string         `yaml:"role_arn,omitempty"`
+	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
+	Port            int            `yaml:"port"`
+}
+
+// Name returns the name of the Lightsail Config.
+func (*LightsailSDConfig) Name() string { return "lightsail" }
+
+// NewDiscoverer returns a Discoverer for the Lightsail Config.
+func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewLightsailDiscovery(c, opts.Logger), nil
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
+func (c *LightsailSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultLightsailSDConfig
+	type plain LightsailSDConfig
+	err := unmarshal((*plain)(c))
+	if err != nil {
+		return err
+	}
+	if c.Region == "" {
+		sess, err := session.NewSession()
+		if err != nil {
+			return err
+		}
+
+		metadata := ec2metadata.New(sess)
+
+		region, err := metadata.Region()
+		if err != nil {
+			return errors.New("Lightsail SD configuration requires a region")
+		}
+		c.Region = region
+	}
+	return nil
+}
+
+// Discovery periodically performs Lightsail-SD requests. It implements
+// the Discoverer interface.
+type LightsailDiscovery struct {
+	*refresh.Discovery
+	cfg       *LightsailSDConfig
+	lightsail *lightsail.Lightsail
+}
+
+// NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery {
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
+	d := &LightsailDiscovery{
+		cfg: conf,
+	}
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"lightsail",
+		time.Duration(d.cfg.RefreshInterval),
+		d.refresh,
+	)
+	return d
+}
+
+func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) {
+	if d.lightsail != nil {
+		return d.lightsail, nil
+	}
+
+	creds := credentials.NewStaticCredentials(d.cfg.AccessKey, string(d.cfg.SecretKey), "")
+	if d.cfg.AccessKey == "" && d.cfg.SecretKey == "" {
+		creds = nil
+	}
+
+	sess, err := session.NewSessionWithOptions(session.Options{
+		Config: aws.Config{
+			Endpoint:    &d.cfg.Endpoint,
+			Region:      &d.cfg.Region,
+			Credentials: creds,
+		},
+		Profile: d.cfg.Profile,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create aws session")
+	}
+
+	if d.cfg.RoleARN != "" {
+		creds := stscreds.NewCredentials(sess, d.cfg.RoleARN)
+		d.lightsail = lightsail.New(sess, &aws.Config{Credentials: creds})
+	} else {
+		d.lightsail = lightsail.New(sess)
+	}
+
+	return d.lightsail, nil
+}
+
+func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	lightsailClient, err := d.lightsailClient()
+	if err != nil {
+		return nil, err
+	}
+
+	tg := &targetgroup.Group{
+		Source: d.cfg.Region,
+	}
+
+	input := &lightsail.GetInstancesInput{}
+
+	output, err := lightsailClient.GetInstancesWithContext(ctx, input)
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok && (awsErr.Code() == "AuthFailure" || awsErr.Code() == "UnauthorizedOperation") {
+			d.lightsail = nil
+		}
+		return nil, errors.Wrap(err, "could not get instances")
+	}
+
+	for _, inst := range output.Instances {
+		if inst.PrivateIpAddress == nil {
+			continue
+		}
+
+		labels := model.LabelSet{
+			lightsailLabelAZ:                  model.LabelValue(*inst.Location.AvailabilityZone),
+			lightsailLabelBlueprintID:         model.LabelValue(*inst.BlueprintId),
+			lightsailLabelBundleID:            model.LabelValue(*inst.BundleId),
+			lightsailLabelInstanceName:        model.LabelValue(*inst.Name),
+			lightsailLabelInstanceState:       model.LabelValue(*inst.State.Name),
+			lightsailLabelInstanceSupportCode: model.LabelValue(*inst.SupportCode),
+			lightsailLabelPrivateIP:           model.LabelValue(*inst.PrivateIpAddress),
+		}
+
+		addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.cfg.Port))
+		labels[model.AddressLabel] = model.LabelValue(addr)
+
+		if inst.PublicIpAddress != nil {
+			labels[lightsailLabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
+		}
+
+		if len(inst.Ipv6Addresses) > 0 {
+			var ipv6addrs []string
+			for _, ipv6addr := range inst.Ipv6Addresses {
+				ipv6addrs = append(ipv6addrs, *ipv6addr)
+			}
+			labels[lightsailLabelIPv6Addresses] = model.LabelValue(
+				lightsailLabelSeparator +
+					strings.Join(ipv6addrs, lightsailLabelSeparator) +
+					lightsailLabelSeparator)
+		}
+
+		for _, t := range inst.Tags {
+			if t == nil || t.Key == nil || t.Value == nil {
+				continue
+			}
+			name := strutil.SanitizeLabelName(*t.Key)
+			labels[lightsailLabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
+		}
+
+		tg.Targets = append(tg.Targets, labels)
+	}
+	return []*targetgroup.Group{tg}, nil
+}

discovery/install/install.go

@@ -16,11 +16,11 @@
 package install
 
 import (
+	_ "github.com/prometheus/prometheus/discovery/aws"          // register aws
 	_ "github.com/prometheus/prometheus/discovery/azure"        // register azure
 	_ "github.com/prometheus/prometheus/discovery/consul"       // register consul
 	_ "github.com/prometheus/prometheus/discovery/digitalocean" // register digitalocean
 	_ "github.com/prometheus/prometheus/discovery/dns"          // register dns
-	_ "github.com/prometheus/prometheus/discovery/ec2"          // register ec2
 	_ "github.com/prometheus/prometheus/discovery/eureka"       // register eureka
 	_ "github.com/prometheus/prometheus/discovery/file"         // register file
 	_ "github.com/prometheus/prometheus/discovery/gce"          // register gce

docs/configuration/configuration.md

@@ -238,6 +238,10 @@ hetzner_sd_configs:
 kubernetes_sd_configs:
   [ - <kubernetes_sd_config> ... ]
 
+# List of Lightsail service discovery configurations.
+lightsail_sd_configs:
+  [ - <lightsail_sd_config> ... ]
+
 # List of Marathon service discovery configurations.
 marathon_sd_configs:
   [ - <marathon_sd_config> ... ]
@@ -1341,6 +1345,54 @@ for a detailed example of configuring Prometheus for Kubernetes.
 You may wish to check out the 3rd party [Prometheus Operator](https://github.com/coreos/prometheus-operator),
 which automates the Prometheus setup on top of Kubernetes.
 
+### `<lightsail_sd_config>`
+
+Lightsail SD configurations allow retrieving scrape targets from [AWS Lightsail](https://aws.amazon.com/lightsail/)
+instances. The private IP address is used by default, but may be changed to
+the public IP address with relabeling.
+
+The following meta labels are available on targets during [relabeling](#relabel_config):
+
+* `__meta_lightsail_availability_zone`: the availability zone in which the instance is running
+* `__meta_lightsail_blueprint_id`: the Lightsail blueprint ID
+* `__meta_lightsail_bundle_id`: the Lightsail bundle ID
+* `__meta_lightsail_instance_name`: the name of the Lightsail instance
+* `__meta_lightsail_instance_state`: the state of the Lightsail instance
+* `__meta_lightsail_instance_support_code`: the support code of the Lightsail instance
+* `__meta_lightsail_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present
+* `__meta_lightsail_private_ip`: the private IP address of the instance
+* `__meta_lightsail_public_ip`: the public IP address of the instance, if available
+* `__meta_lightsail_tag_<tagkey>`: each tag value of the instance
+
+See below for the configuration options for Lightsail discovery:
+
+```yaml
+# The information to access the Lightsail API.
+
+# The AWS region. If blank, the region from the instance metadata is used.
+[ region: <string> ]
+
+# Custom endpoint to be used.
+[ endpoint: <string> ]
+
+# The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID`
+# and `AWS_SECRET_ACCESS_KEY` are used.
+[ access_key: <string> ]
+[ secret_key: <secret> ]
+# Named AWS profile used to connect to the API.
+[ profile: <string> ]
+
+# AWS Role ARN, an alternative to using AWS API keys.
+[ role_arn: <string> ]
+
+# Refresh interval to re-read the instance list.
+[ refresh_interval: <duration> | default = 60s ]
+
+# The port to scrape metrics from. If using the public IP address, this must
+# instead be specified in the relabeling rule.
+[ port: <int> | default = 80 ]
+```
+
 ### `<marathon_sd_config>`
 
 Marathon SD configurations allow retrieving scrape targets using the
@@ -1936,6 +1988,10 @@ hetzner_sd_configs:
 kubernetes_sd_configs:
   [ - <kubernetes_sd_config> ... ]
 
+# List of Lightsail service discovery configurations.
+lightsail_sd_configs:
+        [ - <lightsail_sd_config> ... ]
+
 # List of Marathon service discovery configurations.
 marathon_sd_configs:
   [ - <marathon_sd_config> ... ]