Merge remote-tracking branch 'upstream/main' into sparse-refactor

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,15 +1,17 @@
 <!--
     Don't forget!
-    
+
+    - Please sign CNCF's Developer Certificate of Origin and sign-off your commits by adding the -s / --sign-off flag to `git commit`. See https://github.com/apps/dco for more information.
+
     - If the PR adds or changes a behaviour or fixes a bug of an exported API it would need a unit/e2e test.
-    
+
     - Where possible use only exported APIs for tests to simplify the review and make it as close as possible to an actual library usage.
-    
+
     - No tests are needed for internal implementation changes.
-    
+
     - Performance improvements would need a benchmark test to prove it.
-    
+
     - All exposed objects should have a comment.
-    
+
     - All comments should start with a capital letter and end with a full stop.
- -->
+ -->

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.28.1 / 2021-07-01
+
+* [BUGFIX]: HTTP SD: Allow `charset` specification in `Content-Type` header. #8981
+* [BUGFIX]: HTTP SD: Fix handling of disappeared target groups. #9019
+* [BUGFIX]: Fix incorrect log-level handling after moving to go-kit/log. #9021
+
 ## 2.28.0 / 2021-06-21
 
 * [CHANGE] UI: Make the new experimental PromQL editor the default. #8925

cmd/prometheus/main.go

@@ -131,9 +131,6 @@ type flagConfig struct {
 
 // setFeatureListOptions sets the corresponding options from the featureList.
 func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
-	maxExemplars := c.tsdb.MaxExemplars
-	// Disabled at first. Value from the flag is used if exemplar-storage is set.
-	c.tsdb.MaxExemplars = 0
 	for _, f := range c.featureList {
 		opts := strings.Split(f, ",")
 		for _, o := range opts {
@@ -151,8 +148,8 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 				c.enableExpandExternalLabels = true
 				level.Info(logger).Log("msg", "Experimental expand-external-labels enabled")
 			case "exemplar-storage":
-				c.tsdb.MaxExemplars = maxExemplars
+				c.tsdb.EnableExemplarStorage = true
-				level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled", "maxExemplars", maxExemplars)
+				level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled")
 			case "":
 				continue
 			default:
@@ -259,17 +256,17 @@ func main() {
 	a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
 		SetValue(&newFlagRetentionDuration)
 
-	a.Flag("storage.tsdb.retention.size", "[EXPERIMENTAL] Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases.").
+	a.Flag("storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases.").
 		BytesVar(&cfg.tsdb.MaxBytes)
 
 	a.Flag("storage.tsdb.no-lockfile", "Do not create lockfile in data directory.").
 		Default("false").BoolVar(&cfg.tsdb.NoLockfile)
 
-	a.Flag("storage.tsdb.allow-overlapping-blocks", "[EXPERIMENTAL] Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").
+	a.Flag("storage.tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").
 		Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks)
 
 	a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL.").
-		Default("true").BoolVar(&cfg.tsdb.WALCompression)
+		Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression)
 
 	a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload.").
 		Default("1m").PlaceHolder("<duration>").SetValue(&cfg.RemoteFlushDeadline)
@@ -283,9 +280,6 @@ func main() {
 	a.Flag("storage.remote.read-max-bytes-in-frame", "Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default.").
 		Default("1048576").IntVar(&cfg.web.RemoteReadBytesInFrame)
 
-	a.Flag("storage.exemplars.exemplars-limit", "[EXPERIMENTAL] Maximum number of exemplars to store in in-memory exemplar storage total. 0 disables the exemplar storage. This flag is effective only with --enable-feature=exemplar-storage.").
-		Default("100000").IntVar(&cfg.tsdb.MaxExemplars)
-
 	a.Flag("rules.alert.for-outage-tolerance", "Max time to tolerate prometheus outage for restoring \"for\" state of alert.").
 		Default("1h").SetValue(&cfg.outageTolerance)
 
@@ -316,7 +310,7 @@ func main() {
 	a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return.").
 		Default("50000000").IntVar(&cfg.queryMaxSamples)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage, expand-external-labels. See https://prometheus.io/docs/prometheus/latest/disabled_features/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage, expand-external-labels. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -352,10 +346,18 @@ func main() {
 	}
 
 	// Throw error for invalid config before starting other components.
-	if _, err := config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil {
+	var cfgFile *config.Config
+	if cfgFile, err = config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil {
 		level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "err", err)
 		os.Exit(2)
 	}
+	if cfg.tsdb.EnableExemplarStorage {
+		if cfgFile.StorageConfig.ExemplarsConfig == nil {
+			cfgFile.StorageConfig.ExemplarsConfig = &config.DefaultExemplarsConfig
+		}
+		cfg.tsdb.MaxExemplars = int64(cfgFile.StorageConfig.ExemplarsConfig.MaxExemplars)
+	}
+
 	// Now that the validity of the config is established, set the config
 	// success metrics accordingly, although the config isn't really loaded
 	// yet. This will happen later (including setting these metrics again),
@@ -533,6 +535,9 @@ func main() {
 
 	reloaders := []reloader{
 		{
+			name:     "db_storage",
+			reloader: localStorage.ApplyConfig,
+		}, {
 			name:     "remote_storage",
 			reloader: remoteStorage.ApplyConfig,
 		}, {
@@ -734,11 +739,11 @@ func main() {
 				for {
 					select {
 					case <-hup:
-						if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil {
+						if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
 							level.Error(logger).Log("msg", "Error reloading config", "err", err)
 						}
 					case rc := <-webHandler.Reload():
-						if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil {
+						if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
 							level.Error(logger).Log("msg", "Error reloading config", "err", err)
 							rc <- err
 						} else {
@@ -770,7 +775,7 @@ func main() {
 					return nil
 				}
 
-				if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil {
+				if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
 					return errors.Wrapf(err, "error loading config from %q", cfg.configFile)
 				}
 
@@ -959,7 +964,7 @@ type reloader struct {
 	reloader func(*config.Config) error
 }
 
-func reloadConfig(filename string, expandExternalLabels bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) {
+func reloadConfig(filename string, expandExternalLabels bool, enableExemplarStorage bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) {
 	start := time.Now()
 	timings := []interface{}{}
 	level.Info(logger).Log("msg", "Loading configuration file", "filename", filename)
@@ -978,6 +983,12 @@ func reloadConfig(filename string, expandExternalLabels bool, logger log.Logger,
 		return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename)
 	}
 
+	if enableExemplarStorage {
+		if conf.StorageConfig.ExemplarsConfig == nil {
+			conf.StorageConfig.ExemplarsConfig = &config.DefaultExemplarsConfig
+		}
+	}
+
 	failed := false
 	for _, rl := range rls {
 		rstart := time.Now()
@@ -1083,6 +1094,11 @@ type readyStorage struct {
 	stats           *tsdb.DBStats
 }
 
+func (s *readyStorage) ApplyConfig(conf *config.Config) error {
+	db := s.get()
+	return db.ApplyConfig(conf)
+}
+
 // Set the storage.
 func (s *readyStorage) Set(db *tsdb.DB, startTimeMargin int64) {
 	s.mtx.Lock()
@@ -1262,7 +1278,8 @@ type tsdbOptions struct {
 	StripeSize               int
 	MinBlockDuration         model.Duration
 	MaxBlockDuration         model.Duration
-	MaxExemplars             int
+	EnableExemplarStorage    bool
+	MaxExemplars             int64
 }
 
 func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
@@ -1277,6 +1294,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		StripeSize:               opts.StripeSize,
 		MinBlockDuration:         int64(time.Duration(opts.MinBlockDuration) / time.Millisecond),
 		MaxBlockDuration:         int64(time.Duration(opts.MaxBlockDuration) / time.Millisecond),
+		EnableExemplarStorage:    opts.EnableExemplarStorage,
 		MaxExemplars:             opts.MaxExemplars,
 	}
 }

cmd/promtool/backfill.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"io"
 	"math"
+	"time"
 
 	"github.com/go-kit/log"
 	"github.com/pkg/errors"
@@ -65,8 +66,19 @@ func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) {
 	return maxt, mint, nil
 }
 
-func createBlocks(input []byte, mint, maxt int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) {
+func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) {
 	blockDuration := tsdb.DefaultBlockDuration
+	if maxBlockDuration > tsdb.DefaultBlockDuration {
+		ranges := tsdb.ExponentialBlockRanges(tsdb.DefaultBlockDuration, 10, 3)
+		idx := len(ranges) - 1 // Use largest range if user asked for something enormous.
+		for i, v := range ranges {
+			if v > maxBlockDuration {
+				idx = i - 1
+				break
+			}
+		}
+		blockDuration = ranges[idx]
+	}
 	mint = blockDuration * (mint / blockDuration)
 
 	db, err := tsdb.OpenDBReadOnly(outputDir, nil)
@@ -199,11 +211,11 @@ func createBlocks(input []byte, mint, maxt int64, maxSamplesInAppender int, outp
 
 }
 
-func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool) (err error) {
+func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) {
 	p := textparse.NewOpenMetricsParser(input)
 	maxt, mint, err := getMinAndMaxTimestamps(p)
 	if err != nil {
 		return errors.Wrap(err, "getting min and max timestamp")
 	}
-	return errors.Wrap(createBlocks(input, mint, maxt, maxSamplesInAppender, outputDir, humanReadable, quiet), "block creation")
+	return errors.Wrap(createBlocks(input, mint, maxt, int64(maxBlockDuration/time.Millisecond), maxSamplesInAppender, outputDir, humanReadable, quiet), "block creation")
 }

cmd/promtool/backfill_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"sort"
 	"testing"
+	"time"
 
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
@@ -58,12 +59,12 @@ func queryAllSeries(t testing.TB, q storage.Querier, expectedMinTime, expectedMa
 	return samples
 }
 
-func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime int64, expectedSamples []backfillSample, expectedNumBlocks int) {
+func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, expectedBlockDuration int64, expectedSamples []backfillSample, expectedNumBlocks int) {
 	blocks := db.Blocks()
-	require.Equal(t, expectedNumBlocks, len(blocks))
+	require.Equal(t, expectedNumBlocks, len(blocks), "did not create correct number of blocks")
 
-	for _, block := range blocks {
+	for i, block := range blocks {
-		require.Equal(t, true, block.MinTime()/tsdb.DefaultBlockDuration == (block.MaxTime()-1)/tsdb.DefaultBlockDuration)
+		require.Equal(t, block.MinTime()/expectedBlockDuration, (block.MaxTime()-1)/expectedBlockDuration, "block %d contains data outside of one aligned block duration", i)
 	}
 
 	q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
@@ -75,11 +76,11 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime int6
 	allSamples := queryAllSeries(t, q, expectedMinTime, expectedMaxTime)
 	sortSamples(allSamples)
 	sortSamples(expectedSamples)
-	require.Equal(t, expectedSamples, allSamples)
+	require.Equal(t, expectedSamples, allSamples, "did not create correct samples")
 
 	if len(allSamples) > 0 {
-		require.Equal(t, expectedMinTime, allSamples[0].Timestamp)
+		require.Equal(t, expectedMinTime, allSamples[0].Timestamp, "timestamp of first sample is not the expected minimum time")
-		require.Equal(t, expectedMaxTime, allSamples[len(allSamples)-1].Timestamp)
+		require.Equal(t, expectedMaxTime, allSamples[len(allSamples)-1].Timestamp, "timestamp of last sample is not the expected maximum time")
 	}
 }
 
@@ -89,11 +90,13 @@ func TestBackfill(t *testing.T) {
 		IsOk                 bool
 		Description          string
 		MaxSamplesInAppender int
+		MaxBlockDuration     time.Duration
 		Expected             struct {
-			MinTime   int64
+			MinTime       int64
-			MaxTime   int64
+			MaxTime       int64
-			NumBlocks int
+			NumBlocks     int
-			Samples   []backfillSample
+			BlockDuration int64
+			Samples       []backfillSample
 		}
 	}{
 		{
@@ -102,15 +105,17 @@ func TestBackfill(t *testing.T) {
 			Description:          "Empty file.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   math.MaxInt64,
+				MinTime:       math.MaxInt64,
-				MaxTime:   math.MinInt64,
+				MaxTime:       math.MinInt64,
-				NumBlocks: 0,
+				NumBlocks:     0,
-				Samples:   []backfillSample{},
+				BlockDuration: tsdb.DefaultBlockDuration,
+				Samples:       []backfillSample{},
 			},
 		},
 		{
@@ -124,14 +129,16 @@ http_requests_total{code="400"} 1 1565133713.990
 			Description:          "Multiple samples with different timestamp for different series.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565133713990,
+				MaxTime:       1565133713990,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -158,14 +165,16 @@ http_requests_total{code="200"} 1023 1565652113.989
 			Description:          "Multiple samples separated by 3 days.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565652113989,
+				MaxTime:       1565652113989,
-				NumBlocks: 3,
+				NumBlocks:     3,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -196,14 +205,16 @@ http_requests_total{code="200"} 1021 1565133713.989
 			Description:          "Unordered samples from multiple series, which end in different blocks.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565392913989,
+				MaxTime:       1565392913989,
-				NumBlocks: 2,
+				NumBlocks:     2,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -230,14 +241,16 @@ http_requests_total{code="400"} 2 1565133715.989
 			Description:          "Multiple samples with different timestamp for the same series.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565133715989,
+				MaxTime:       1565133715989,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -260,6 +273,132 @@ http_requests_total{code="400"} 2 1565133715.989
 		{
 			ToParse: `# HELP http_requests_total The total number of HTTP requests.
 # TYPE http_requests_total counter
+http_requests_total{code="200"} 1021 1624463088.000
+http_requests_total{code="200"} 1 1627055153.000
+http_requests_total{code="400"} 2 1627056153.000
+# EOF
+`,
+			IsOk:                 true,
+			Description:          "Long maximum block duration puts all data into one block.",
+			MaxSamplesInAppender: 5000,
+			MaxBlockDuration:     2048 * time.Hour,
+			Expected: struct {
+				MinTime       int64
+				MaxTime       int64
+				NumBlocks     int
+				BlockDuration int64
+				Samples       []backfillSample
+			}{
+				MinTime:       1624463088000,
+				MaxTime:       1627056153000,
+				NumBlocks:     1,
+				BlockDuration: int64(1458 * time.Hour / time.Millisecond),
+				Samples: []backfillSample{
+					{
+						Timestamp: 1624463088000,
+						Value:     1021,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1627055153000,
+						Value:     1,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1627056153000,
+						Value:     2,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "400"),
+					},
+				},
+			},
+		},
+		{
+			ToParse: `# HELP http_requests_total The total number of HTTP requests.
+# TYPE http_requests_total counter
+http_requests_total{code="200"} 1 1624463088.000
+http_requests_total{code="200"} 2 1629503088.000
+http_requests_total{code="200"} 3 1629863088.000
+# EOF
+`,
+			IsOk:                 true,
+			Description:          "Long maximum block duration puts all data into two blocks.",
+			MaxSamplesInAppender: 5000,
+			MaxBlockDuration:     2048 * time.Hour,
+			Expected: struct {
+				MinTime       int64
+				MaxTime       int64
+				NumBlocks     int
+				BlockDuration int64
+				Samples       []backfillSample
+			}{
+				MinTime:       1624463088000,
+				MaxTime:       1629863088000,
+				NumBlocks:     2,
+				BlockDuration: int64(1458 * time.Hour / time.Millisecond),
+				Samples: []backfillSample{
+					{
+						Timestamp: 1624463088000,
+						Value:     1,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1629503088000,
+						Value:     2,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1629863088000,
+						Value:     3,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+				},
+			},
+		},
+		{
+			ToParse: `# HELP http_requests_total The total number of HTTP requests.
+# TYPE http_requests_total counter
+http_requests_total{code="200"} 1 1624463088.000
+http_requests_total{code="200"} 2 1765943088.000
+http_requests_total{code="200"} 3 1768463088.000
+# EOF
+`,
+			IsOk:                 true,
+			Description:          "Maximum block duration longer than longest possible duration, uses largest duration, puts all data into two blocks.",
+			MaxSamplesInAppender: 5000,
+			MaxBlockDuration:     200000 * time.Hour,
+			Expected: struct {
+				MinTime       int64
+				MaxTime       int64
+				NumBlocks     int
+				BlockDuration int64
+				Samples       []backfillSample
+			}{
+				MinTime:       1624463088000,
+				MaxTime:       1768463088000,
+				NumBlocks:     2,
+				BlockDuration: int64(39366 * time.Hour / time.Millisecond),
+				Samples: []backfillSample{
+					{
+						Timestamp: 1624463088000,
+						Value:     1,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1765943088000,
+						Value:     2,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+					{
+						Timestamp: 1768463088000,
+						Value:     3,
+						Labels:    labels.FromStrings("__name__", "http_requests_total", "code", "200"),
+					},
+				},
+			},
+		},
+		{
+			ToParse: `# HELP http_requests_total The total number of HTTP requests.
+# TYPE http_requests_total counter
 http_requests_total{code="200"} 1021 1565133713.989
 http_requests_total{code="200"} 1022 1565144513.989
 http_requests_total{code="400"} 2 1565155313.989
@@ -270,14 +409,16 @@ http_requests_total{code="400"} 1 1565166113.989
 			Description:          "Multiple samples that end up in different blocks.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565166113989,
+				MaxTime:       1565166113989,
-				NumBlocks: 4,
+				NumBlocks:     4,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -318,14 +459,16 @@ http_requests_total{code="400"} 1 1565166113.989
 			Description:          "Number of samples are greater than the sample batch size.",
 			MaxSamplesInAppender: 2,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1565133713989,
+				MinTime:       1565133713989,
-				MaxTime:   1565166113989,
+				MaxTime:       1565166113989,
-				NumBlocks: 4,
+				NumBlocks:     4,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1565133713989,
@@ -378,14 +521,16 @@ http_requests_total{code="400"} 1024 7199
 			Description:          "One series spanning 2h in same block should not cause problems to other series.",
 			MaxSamplesInAppender: 1,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   0,
+				MinTime:       0,
-				MaxTime:   7199000,
+				MaxTime:       7199000,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 0,
@@ -418,14 +563,16 @@ http_requests_total{code="400"} 1024 7199
 			Description:          "Sample with no #HELP or #TYPE keyword.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   6900000,
+				MinTime:       6900000,
-				MaxTime:   6900000,
+				MaxTime:       6900000,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 6900000,
@@ -442,14 +589,16 @@ http_requests_total{code="400"} 1024 7199
 			Description:          "Sample without newline after # EOF.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   6900000,
+				MinTime:       6900000,
-				MaxTime:   6900000,
+				MaxTime:       6900000,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 6900000,
@@ -467,14 +616,16 @@ http_requests_total{code="400"} 1024 7199
 			Description:          "Bare sample.",
 			MaxSamplesInAppender: 5000,
 			Expected: struct {
-				MinTime   int64
+				MinTime       int64
-				MaxTime   int64
+				MaxTime       int64
-				NumBlocks int
+				NumBlocks     int
-				Samples   []backfillSample
+				BlockDuration int64
+				Samples       []backfillSample
 			}{
-				MinTime:   1001000,
+				MinTime:       1001000,
-				MaxTime:   1001000,
+				MaxTime:       1001000,
-				NumBlocks: 1,
+				NumBlocks:     1,
+				BlockDuration: tsdb.DefaultBlockDuration,
 				Samples: []backfillSample{
 					{
 						Timestamp: 1001000,
@@ -532,28 +683,32 @@ after_eof 1 2
 		},
 	}
 	for _, test := range tests {
-		t.Logf("Test:%s", test.Description)
+		t.Run(test.Description, func(t *testing.T) {
+			t.Logf("Test:%s", test.Description)
 
-		outputDir, err := ioutil.TempDir("", "myDir")
+			outputDir, err := ioutil.TempDir("", "myDir")
-		require.NoError(t, err)
+			require.NoError(t, err)
-		defer func() {
+			defer func() {
-			require.NoError(t, os.RemoveAll(outputDir))
+				require.NoError(t, os.RemoveAll(outputDir))
-		}()
+			}()
 
-		err = backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false)
+			err = backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false, test.MaxBlockDuration)
 
-		if !test.IsOk {
+			if !test.IsOk {
-			require.Error(t, err, test.Description)
+				require.Error(t, err, test.Description)
-			continue
+				return
-		}
+			}
 
-		require.NoError(t, err)
+			require.NoError(t, err)
-		db, err := tsdb.Open(outputDir, nil, nil, tsdb.DefaultOptions(), nil)
+			options := tsdb.DefaultOptions()
-		require.NoError(t, err)
+			options.RetentionDuration = int64(10 * 365 * 24 * time.Hour / time.Millisecond) // maximum duration tests require a long retention
-		defer func() {
+			db, err := tsdb.Open(outputDir, nil, nil, options, nil)
-			require.NoError(t, db.Close())
+			require.NoError(t, err)
-		}()
+			defer func() {
+				require.NoError(t, db.Close())
+			}()
 
-		testBlocks(t, db, test.Expected.MinTime, test.Expected.MaxTime, test.Expected.Samples, test.Expected.NumBlocks)
+			testBlocks(t, db, test.Expected.MinTime, test.Expected.MaxTime, test.Expected.BlockDuration, test.Expected.Samples, test.Expected.NumBlocks)
+		})
 	}
 }

cmd/promtool/main.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math"
 	"net/http"
 	"net/url"
@@ -40,12 +41,15 @@ import (
 	"github.com/prometheus/common/version"
 	"github.com/prometheus/exporter-toolkit/web"
 	"gopkg.in/alecthomas/kingpin.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery/file"
 	_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
 	"github.com/prometheus/prometheus/discovery/kubernetes"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
+	"github.com/prometheus/prometheus/promql"
 )
 
 func main() {
@@ -128,7 +132,7 @@ func main() {
 	benchWriteNumScrapes := tsdbBenchWriteCmd.Flag("scrapes", "Number of scrapes to simulate.").Default("3000").Int()
 	benchSamplesFile := tsdbBenchWriteCmd.Arg("file", "Input file with samples data, default is ("+filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")+").").Default(filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")).String()
 
-	tsdbAnalyzeCmd := tsdbCmd.Command("analyze", "Analyze churn, label pair cardinality.")
+	tsdbAnalyzeCmd := tsdbCmd.Command("analyze", "Analyze churn, label pair cardinality and compaction efficiency.")
 	analyzePath := tsdbAnalyzeCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
 	analyzeBlockID := tsdbAnalyzeCmd.Arg("block id", "Block to analyze (default is the last block).").String()
 	analyzeLimit := tsdbAnalyzeCmd.Flag("limit", "How many items to show in each list.").Default("20").Int()
@@ -145,8 +149,8 @@ func main() {
 	importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.")
 	importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
 	importQuiet := importCmd.Flag("quiet", "Do not print created blocks.").Short('q').Bool()
+	maxBlockDuration := importCmd.Flag("max-block-duration", "Maximum duration created blocks may span. Anything less than 2h is ignored.").Hidden().PlaceHolder("<duration>").Duration()
 	openMetricsImportCmd := importCmd.Command("openmetrics", "Import samples from OpenMetrics input and produce TSDB blocks. Please refer to the storage docs for more details.")
-	// TODO(aSquare14): add flag to set default block duration
 	importFilePath := openMetricsImportCmd.Arg("input file", "OpenMetrics file to read samples from.").Required().String()
 	importDBPath := openMetricsImportCmd.Arg("output directory", "Output directory for generated blocks.").Default(defaultDBPath).String()
 	importRulesCmd := importCmd.Command("rules", "Create blocks of data for new recording rules.")
@@ -162,6 +166,8 @@ func main() {
 		"A list of one or more files containing recording rules to be backfilled. All recording rules listed in the files will be backfilled. Alerting rules are not evaluated.",
 	).Required().ExistingFiles()
 
+	featureList := app.Flag("enable-feature", "Comma separated feature names to enable (only PromQL related). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details.").Default("").Strings()
+
 	parsedCmd := kingpin.MustParse(app.Parse(os.Args[1:]))
 
 	var p printer
@@ -172,6 +178,23 @@ func main() {
 		p = &promqlPrinter{}
 	}
 
+	var queryOpts promql.LazyLoaderOpts
+	for _, f := range *featureList {
+		opts := strings.Split(f, ",")
+		for _, o := range opts {
+			switch o {
+			case "promql-at-modifier":
+				queryOpts.EnableAtModifier = true
+			case "promql-negative-offset":
+				queryOpts.EnableNegativeOffset = true
+			case "":
+				continue
+			default:
+				fmt.Printf("  WARNING: Unknown option for --enable-feature: %q\n", o)
+			}
+		}
+	}
+
 	switch parsedCmd {
 	case checkConfigCmd.FullCommand():
 		os.Exit(CheckConfig(*configFiles...))
@@ -207,7 +230,7 @@ func main() {
 		os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
 
 	case testRulesCmd.FullCommand():
-		os.Exit(RulesUnitTest(*testRulesFiles...))
+		os.Exit(RulesUnitTest(queryOpts, *testRulesFiles...))
 
 	case tsdbBenchWriteCmd.FullCommand():
 		os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics, *benchWriteNumScrapes)))
@@ -222,7 +245,7 @@ func main() {
 		os.Exit(checkErr(dumpSamples(*dumpPath, *dumpMinTime, *dumpMaxTime)))
 	//TODO(aSquare14): Work on adding support for custom block size.
 	case openMetricsImportCmd.FullCommand():
-		os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet))
+		os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration))
 
 	case importRulesCmd.FullCommand():
 		os.Exit(checkErr(importRules(*importRulesURL, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *importRulesFiles...)))
@@ -337,8 +360,12 @@ func checkConfig(filename string) ([]string, error) {
 						return nil, err
 					}
 					if len(files) != 0 {
-						// There was at least one match for the glob and we can assume checkFileExists
+						for _, f := range files {
-						// for all matches would pass, we can continue the loop.
+							err = checkSDFile(f)
+							if err != nil {
+								return nil, errors.Errorf("checking SD file %q: %v", file, err)
+							}
+						}
 						continue
 					}
 					fmt.Printf("  WARNING: file %q for file_sd in scrape job %q does not exist\n", file, scfg.JobName)
@@ -368,6 +395,42 @@ func checkTLSConfig(tlsConfig config_util.TLSConfig) error {
 	return nil
 }
 
+func checkSDFile(filename string) error {
+	fd, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	content, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return err
+	}
+
+	var targetGroups []*targetgroup.Group
+
+	switch ext := filepath.Ext(filename); strings.ToLower(ext) {
+	case ".json":
+		if err := json.Unmarshal(content, &targetGroups); err != nil {
+			return err
+		}
+	case ".yml", ".yaml":
+		if err := yaml.UnmarshalStrict(content, &targetGroups); err != nil {
+			return err
+		}
+	default:
+		return errors.Errorf("invalid file extension: %q", ext)
+	}
+
+	for i, tg := range targetGroups {
+		if tg == nil {
+			return errors.Errorf("nil target group item found (index %d)", i)
+		}
+	}
+
+	return nil
+}
+
 // CheckRules validates rule files.
 func CheckRules(files ...string) int {
 	failed := false

cmd/promtool/main_test.go

@@ -77,3 +77,44 @@ func mockServer(code int, body string) (*httptest.Server, func() *http.Request)
 	}
 	return server, f
 }
+
+func TestCheckSDFile(t *testing.T) {
+	cases := []struct {
+		name string
+		file string
+		err  string
+	}{
+		{
+			name: "good .yml",
+			file: "./testdata/good-sd-file.yml",
+		},
+		{
+			name: "good .yaml",
+			file: "./testdata/good-sd-file.yaml",
+		},
+		{
+			name: "good .json",
+			file: "./testdata/good-sd-file.json",
+		},
+		{
+			name: "bad file extension",
+			file: "./testdata/bad-sd-file-extension.nonexistant",
+			err:  "invalid file extension: \".nonexistant\"",
+		},
+		{
+			name: "bad format",
+			file: "./testdata/bad-sd-file-format.yml",
+			err:  "yaml: unmarshal errors:\n  line 1: field targats not found in type struct { Targets []string \"yaml:\\\"targets\\\"\"; Labels model.LabelSet \"yaml:\\\"labels\\\"\" }",
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			err := checkSDFile(test.file)
+			if test.err != "" {
+				require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error())
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}

cmd/promtool/rules_test.go

@@ -91,7 +91,7 @@ func TestBackfillRuleIntegration(t *testing.T) {
 				group1 := ruleImporter.groups[path1+";group0"]
 				require.NotNil(t, group1)
 				const defaultInterval = 60
-				require.Equal(t, time.Duration(defaultInterval*time.Second), group1.Interval())
+				require.Equal(t, defaultInterval*time.Second, group1.Interval())
 				gRules := group1.Rules()
 				require.Equal(t, 1, len(gRules))
 				require.Equal(t, "rule1", gRules[0].Name())
@@ -100,7 +100,7 @@ func TestBackfillRuleIntegration(t *testing.T) {
 
 				group2 := ruleImporter.groups[path2+";group2"]
 				require.NotNil(t, group2)
-				require.Equal(t, time.Duration(defaultInterval*time.Second), group2.Interval())
+				require.Equal(t, defaultInterval*time.Second, group2.Interval())
 				g2Rules := group2.Rules()
 				require.Equal(t, 2, len(g2Rules))
 				require.Equal(t, "grp2_rule1", g2Rules[0].Name())

cmd/promtool/testdata/at-modifier-test.yml

@@ -0,0 +1,7 @@
+rule_files:
+  - at-modifier.yml
+
+tests:
+  - input_series:
+      - series: "requests{}"
+        values: 1

cmd/promtool/testdata/at-modifier.yml

@@ -0,0 +1,7 @@
+# This is the rules file for at-modifier-test.yml.
+
+groups:
+  - name: at-modifier
+    rules:
+      - record: x
+        expr: "requests @ 1000"

cmd/promtool/testdata/bad-sd-file-format.yml

@@ -0,0 +1,2 @@
+- targats:
+    - localhost:9100

cmd/promtool/testdata/good-sd-file.json

@@ -0,0 +1,8 @@
+[
+  {
+    "labels": {
+      "job": "node"
+    },
+    "targets": ["localhost:9100"]
+  }
+]

cmd/promtool/testdata/good-sd-file.yaml

@@ -0,0 +1,4 @@
+- labels:
+    job: node
+- targets:
+    - localhost:9100

cmd/promtool/testdata/good-sd-file.yml

@@ -0,0 +1,4 @@
+- labels:
+    job: node
+- targets:
+    - localhost:9100

cmd/promtool/testdata/negative-offset-test.yml

@@ -0,0 +1,7 @@
+rule_files:
+  - negative-offset.yml
+
+tests:
+  - input_series:
+      - series: "requests{}"
+        values: 1

cmd/promtool/testdata/negative-offset.yml

@@ -0,0 +1,7 @@
+# This is the rules file for negative-offset-test.yml.
+
+groups:
+  - name: negative-offset
+    rules:
+      - record: x
+        expr: "requests offset -5m"

cmd/promtool/tsdb.go

@@ -17,8 +17,10 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/prometheus/prometheus/tsdb/index"
 	"io"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -561,6 +563,60 @@ func analyzeBlock(path, blockID string, limit int) error {
 	}
 	fmt.Printf("\nHighest cardinality metric names:\n")
 	printInfo(postingInfos)
+
+	return analyzeCompaction(block, ir)
+}
+
+func analyzeCompaction(block tsdb.BlockReader, indexr tsdb.IndexReader) (err error) {
+	postingsr, err := indexr.Postings(index.AllPostingsKey())
+	if err != nil {
+		return err
+	}
+	chunkr, err := block.Chunks()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = tsdb_errors.NewMulti(err, chunkr.Close()).Err()
+	}()
+
+	const maxSamplesPerChunk = 120
+	nBuckets := 10
+	histogram := make([]int, nBuckets)
+	totalChunks := 0
+	for postingsr.Next() {
+		var lbsl = labels.Labels{}
+		var chks []chunks.Meta
+		if err := indexr.Series(postingsr.At(), &lbsl, &chks); err != nil {
+			return err
+		}
+
+		for _, chk := range chks {
+			// Load the actual data of the chunk.
+			chk, err := chunkr.Chunk(chk.Ref)
+			if err != nil {
+				return err
+			}
+			chunkSize := math.Min(float64(chk.NumSamples()), maxSamplesPerChunk)
+			// Calculate the bucket for the chunk and increment it in the histogram.
+			bucket := int(math.Ceil(float64(nBuckets)*chunkSize/maxSamplesPerChunk)) - 1
+			histogram[bucket]++
+			totalChunks++
+		}
+	}
+
+	fmt.Printf("\nCompaction analysis:\n")
+	fmt.Println("Fullness: Amount of samples in chunks (100% is 120 samples)")
+	// Normalize absolute counts to percentages and print them out.
+	for bucket, count := range histogram {
+		percentage := 100.0 * count / totalChunks
+		fmt.Printf("%7d%%: ", (bucket+1)*10)
+		for j := 0; j < percentage; j++ {
+			fmt.Printf("#")
+		}
+		fmt.Println()
+	}
+
 	return nil
 }
 
@@ -611,7 +667,7 @@ func checkErr(err error) int {
 	return 0
 }
 
-func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool) int {
+func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) int {
 	inputFile, err := fileutil.OpenMmapFile(path)
 	if err != nil {
 		return checkErr(err)
@@ -622,5 +678,5 @@ func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet boo
 		return checkErr(errors.Wrap(err, "create output dir"))
 	}
 
-	return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet))
+	return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet, maxBlockDuration))
 }

cmd/promtool/unittest.go

@@ -39,11 +39,11 @@ import (
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(files ...string) int {
+func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int {
 	failed := false
 
 	for _, f := range files {
-		if errs := ruleUnitTest(f); errs != nil {
+		if errs := ruleUnitTest(f, queryOpts); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")
 			for _, e := range errs {
 				fmt.Fprintln(os.Stderr, e.Error())
@@ -60,7 +60,7 @@ func RulesUnitTest(files ...string) int {
 	return 0
 }
 
-func ruleUnitTest(filename string) []error {
+func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error {
 	fmt.Println("Unit Testing: ", filename)
 
 	b, err := ioutil.ReadFile(filename)
@@ -95,7 +95,7 @@ func ruleUnitTest(filename string) []error {
 	// Testing.
 	var errs []error
 	for _, t := range unitTestInp.Tests {
-		ers := t.test(evalInterval, groupOrderMap, unitTestInp.RuleFiles...)
+		ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...)
 		if ers != nil {
 			errs = append(errs, ers...)
 		}
@@ -151,9 +151,9 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, ruleFiles ...string) []error {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error {
 	// Setup testing suite.
-	suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString())
+	suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
 	if err != nil {
 		return []error{err}
 	}
@@ -255,7 +255,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 
 		for {
 			if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) &&
-				time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) {
+				time.Duration(alertEvalTimes[curr]) < ts.Add(evalInterval).Sub(mint)) {
 				break
 			}
 

cmd/promtool/unittest_test.go

@@ -13,16 +13,21 @@
 
 package main
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/prometheus/prometheus/promql"
+)
 
 func TestRulesUnitTest(t *testing.T) {
 	type args struct {
 		files []string
 	}
 	tests := []struct {
-		name string
+		name      string
-		args args
+		args      args
-		want int
+		queryOpts promql.LazyLoaderOpts
+		want      int
 	}{
 		{
 			name: "Passing Unit Tests",
@@ -66,10 +71,44 @@ func TestRulesUnitTest(t *testing.T) {
 			},
 			want: 1,
 		},
+		{
+			name: "Disabled feature (@ modifier)",
+			args: args{
+				files: []string{"./testdata/at-modifier-test.yml"},
+			},
+			want: 1,
+		},
+		{
+			name: "Enabled feature (@ modifier)",
+			args: args{
+				files: []string{"./testdata/at-modifier-test.yml"},
+			},
+			queryOpts: promql.LazyLoaderOpts{
+				EnableAtModifier: true,
+			},
+			want: 0,
+		},
+		{
+			name: "Disabled feature (negative offset)",
+			args: args{
+				files: []string{"./testdata/negative-offset-test.yml"},
+			},
+			want: 1,
+		},
+		{
+			name: "Enabled feature (negative offset)",
+			args: args{
+				files: []string{"./testdata/negative-offset-test.yml"},
+			},
+			queryOpts: promql.LazyLoaderOpts{
+				EnableNegativeOffset: true,
+			},
+			want: 0,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})

config/config.go

@@ -183,6 +183,15 @@ var (
 		RemoteTimeout:    model.Duration(1 * time.Minute),
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
 	}
+
+	// DefaultStorageConfig is the default TSDB/Exemplar storage configuration.
+	DefaultStorageConfig = StorageConfig{
+		ExemplarsConfig: &DefaultExemplarsConfig,
+	}
+
+	DefaultExemplarsConfig = ExemplarsConfig{
+		MaxExemplars: 100000,
+	}
 )
 
 // Config is the top-level configuration for Prometheus's config files.
@@ -191,6 +200,7 @@ type Config struct {
 	AlertingConfig AlertingConfig  `yaml:"alerting,omitempty"`
 	RuleFiles      []string        `yaml:"rule_files,omitempty"`
 	ScrapeConfigs  []*ScrapeConfig `yaml:"scrape_configs,omitempty"`
+	StorageConfig  StorageConfig   `yaml:"storage,omitempty"`
 
 	RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
 	RemoteReadConfigs  []*RemoteReadConfig  `yaml:"remote_read,omitempty"`
@@ -464,6 +474,18 @@ func (c *ScrapeConfig) MarshalYAML() (interface{}, error) {
 	return discovery.MarshalYAMLWithInlineConfigs(c)
 }
 
+// StorageConfig configures runtime reloadable configuration options.
+type StorageConfig struct {
+	ExemplarsConfig *ExemplarsConfig `yaml:"exemplars,omitempty"`
+}
+
+// ExemplarsConfig configures runtime reloadable configuration options.
+type ExemplarsConfig struct {
+	// MaxExemplars sets the size, in # of exemplars stored, of the single circular buffer used to store exemplars in memory.
+	// Use a value of 0 or less than 0 to disable the storage without having to restart Prometheus.
+	MaxExemplars int64 `yaml:"max_exemplars,omitempty"`
+}
+
 // AlertingConfig configures alerting and alertmanager related configs.
 type AlertingConfig struct {
 	AlertRelabelConfigs []*relabel.Config   `yaml:"alert_relabel_configs,omitempty"`

config/config_test.go

@@ -48,6 +48,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/scaleway"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/discovery/triton"
+	"github.com/prometheus/prometheus/discovery/xds"
 	"github.com/prometheus/prometheus/discovery/zookeeper"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/relabel"
@@ -439,6 +440,26 @@ var expectedConf = &Config{
 				},
 			},
 		},
+		{
+			JobName: "service-kuma",
+
+			HonorTimestamps: true,
+			ScrapeInterval:  model.Duration(15 * time.Second),
+			ScrapeTimeout:   DefaultGlobalConfig.ScrapeTimeout,
+
+			MetricsPath:      DefaultScrapeConfig.MetricsPath,
+			Scheme:           DefaultScrapeConfig.Scheme,
+			HTTPClientConfig: config.DefaultHTTPClientConfig,
+
+			ServiceDiscoveryConfigs: discovery.Configs{
+				&xds.KumaSDConfig{
+					Server:           "http://kuma-control-plane.kuma-system.svc:5676",
+					HTTPClientConfig: config.DefaultHTTPClientConfig,
+					RefreshInterval:  model.Duration(15 * time.Second),
+					FetchTimeout:     model.Duration(2 * time.Minute),
+				},
+			},
+		},
 		{
 			JobName: "service-marathon",
 
@@ -714,11 +735,12 @@ var expectedConf = &Config{
 
 			ServiceDiscoveryConfigs: discovery.Configs{
 				&moby.DockerSDConfig{
-					Filters:          []moby.Filter{},
+					Filters:            []moby.Filter{},
-					Host:             "unix:///var/run/docker.sock",
+					Host:               "unix:///var/run/docker.sock",
-					Port:             80,
+					Port:               80,
-					RefreshInterval:  model.Duration(60 * time.Second),
+					HostNetworkingHost: "localhost",
-					HTTPClientConfig: config.DefaultHTTPClientConfig,
+					RefreshInterval:    model.Duration(60 * time.Second),
+					HTTPClientConfig:   config.DefaultHTTPClientConfig,
 				},
 			},
 		},

config/testdata/conf.good.yml

@@ -109,7 +109,6 @@ scrape_configs:
           - second.dns.address.domain.com
       - names:
           - first.dns.address.domain.com
-        # refresh_interval defaults to 30s.
 
     relabel_configs:
       - source_labels: [job]
@@ -193,6 +192,11 @@ scrape_configs:
       username: "myusername"
       password_file: valid_password_file
 
+  - job_name: service-kuma
+
+    kuma_sd_configs:
+      - server: http://kuma-control-plane.kuma-system.svc:5676
+
   - job_name: service-marathon
     marathon_sd_configs:
       - servers:

config/testdata/roundtrip.good.yml

@@ -106,6 +106,9 @@ scrape_configs:
           username: username
           password_file: valid_password_file
 
+    kuma_sd_configs:
+      - server: http://kuma-control-plane.kuma-system.svc:5676
+
     marathon_sd_configs:
       - servers:
           - https://marathon.example.com:443

consoles/node-disk.html

@@ -1,5 +1,44 @@
 {{ template "head" . }}
 
+{{ template "prom_right_table_head" }}
+<tr>
+  <th colspan="2">Disks</th>
+</tr>
+{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
+  <th colspan="2">{{ .Labels.device }}</th>
+  <tr>
+    <td>Utilization</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>Throughput</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Avg Read Time</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Avg Write Time</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Filesystem Fullness</th>
+</tr>
+{{ define "roughlyNearZero" }}
+{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
+{{ end }}
+{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
+  <tr>
+    <td>{{ .Labels.mountpoint }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
+  </tr>
+{{ end }}
+<tr>
+</tr>
+{{ template "prom_right_table_tail" }}
+
 {{ template "prom_content_head" . }}
   <h1>Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
 
@@ -34,44 +73,6 @@
     yTitle: 'Filesystem Fullness'
   })
   </script>
-
-{{ template "prom_right_table_head" }}
-  <th colspan="2">Disks</th>
-</tr>
-{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
-  <th colspan="2">{{ .Labels.device }}</th>
-  <tr>
-    <td>Utilization</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
-  </tr>
-  <tr>
-    <td>Throughput</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-  <tr>
-    <td>Avg Read Time</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
-  </tr>
-  <tr>
-    <td>Avg Write Time</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
-  </tr>
-{{ end }}
-  <th colspan="2">Filesystem Fullness</th>
-</tr>
-{{ define "roughlyNearZero" }}
-{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
-{{ end }}
-{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
-  <tr>
-    <td>{{ .Labels.mountpoint }}</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
-  </tr>
-{{ end }}
-<tr>
-</tr>
-{{ template "prom_right_table_tail" }}
-
 {{ template "prom_content_tail" . }}
 
 {{ template "tail" }}

consoles/node-overview.html

@@ -1,5 +1,66 @@
 {{ template "head" . }}
 
+{{ template "prom_right_table_head" }}
+  <tr><th colspan="2">Overview</th></tr>
+  <tr>
+    <td>User CPU</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>System CPU</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>Memory Total</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+  </tr>
+  <tr>
+    <td>Memory Free</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+  </tr>
+<tr>
+  <th colspan="2">Network</th>
+</tr>
+{{ range printf "node_network_receive_bytes_total{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Received</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>{{ .Labels.device }} Transmitted</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Disks</th>
+</tr>
+{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Utilization</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+  </tr>
+{{ end }}
+{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Throughput</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Filesystem Fullness</th>
+</tr>
+{{ define "roughlyNearZero" }}
+{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
+{{ end }}
+{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
+  <tr>
+    <td>{{ .Labels.mountpoint }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
+  </tr>
+{{ end }}
+
+{{ template "prom_right_table_tail" }}
+
 {{ template "prom_content_head" . }}
   <h1>Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
 
@@ -55,68 +116,6 @@
   })
   </script>
 
-{{ template "prom_right_table_head" }}
-  <tr><th colspan="2">Overview</th></tr>
-  <tr>
-    <td>User CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
-  </tr>
-  <tr>
-    <td>System CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
-  </tr>
-  <tr>
-    <td>Memory Total</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
-  </tr>
-  <tr>
-    <td>Memory Free</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
-  </tr>
-<tr>
-  <th colspan="2">Network</th>
-</tr>
-{{ range printf "node_network_receive_bytes_total{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Received</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-  <tr>
-    <td>{{ .Labels.device }} Transmitted</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-{{ end }}
-</tr>
-<tr>
-  <th colspan="2">Disks</th>
-</tr>
-{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Utilization</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
-  </tr>
-{{ end }}
-{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Throughput</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-{{ end }}
-<tr>
-  <th colspan="2">Filesystem Fullness</th>
-</tr>
-{{ define "roughlyNearZero" }}
-{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
-{{ end }}
-{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
-  <tr>
-    <td>{{ .Labels.mountpoint }}</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
-  </tr>
-{{ end }}
-</tr>
-{{ template "prom_right_table_tail" }}
-
 {{ template "prom_content_tail" . }}
 
 {{ template "tail" }}

consoles/node.html

@@ -27,6 +27,7 @@
 {{ else }}
 <tr><td colspan=4>No nodes found.</td></tr>
 {{ end }}
+</table>
 
 
 {{ template "prom_content_tail" . }}

discovery/aws/ec2.go

@@ -28,6 +28,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -42,6 +43,7 @@ const (
 	ec2Label                  = model.MetaLabelPrefix + "ec2_"
 	ec2LabelAMI               = ec2Label + "ami"
 	ec2LabelAZ                = ec2Label + "availability_zone"
+	ec2LabelAZID              = ec2Label + "availability_zone_id"
 	ec2LabelArch              = ec2Label + "architecture"
 	ec2LabelIPv6Addresses     = ec2Label + "ipv6_addresses"
 	ec2LabelInstanceID        = ec2Label + "instance_id"
@@ -132,8 +134,14 @@ func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type EC2Discovery struct {
 	*refresh.Discovery
-	cfg *EC2SDConfig
+	logger log.Logger
-	ec2 *ec2.EC2
+	cfg    *EC2SDConfig
+	ec2    *ec2.EC2
+
+	// azToAZID maps this account's availability zones to their underlying AZ
+	// ID, e.g. eu-west-2a -> euw2-az2. Refreshes are performed sequentially, so
+	// no locking is required.
+	azToAZID map[string]string
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
@@ -142,7 +150,8 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 		logger = log.NewNopLogger()
 	}
 	d := &EC2Discovery{
-		cfg: conf,
+		logger: logger,
+		cfg:    conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
 		logger,
@@ -153,7 +162,7 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 	return d
 }
 
-func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) {
+func (d *EC2Discovery) ec2Client(ctx context.Context) (*ec2.EC2, error) {
 	if d.ec2 != nil {
 		return d.ec2, nil
 	}
@@ -185,8 +194,28 @@ func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) {
 	return d.ec2, nil
 }
 
+func (d *EC2Discovery) azID(ctx context.Context, az string) (string, error) {
+	if azID, ok := d.azToAZID[az]; ok {
+		return azID, nil
+	}
+
+	azs, err := d.ec2.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{})
+	if err != nil {
+		return "", errors.Wrap(err, "could not describe availability zones")
+	}
+	d.azToAZID = make(map[string]string, len(azs.AvailabilityZones))
+	for _, az := range azs.AvailabilityZones {
+		d.azToAZID[*az.ZoneName] = *az.ZoneId
+	}
+
+	if azID, ok := d.azToAZID[az]; ok {
+		return azID, nil
+	}
+	return "", fmt.Errorf("no availability zone ID mapping found for %s", az)
+}
+
 func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
-	ec2Client, err := d.ec2Client()
+	ec2Client, err := d.ec2Client(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -211,6 +240,14 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 				if inst.PrivateIpAddress == nil {
 					continue
 				}
+				azID, err := d.azID(ctx, *inst.Placement.AvailabilityZone)
+				if err != nil {
+					level.Warn(d.logger).Log(
+						"msg", "Unable to determine availability zone ID",
+						"az", *inst.Placement.AvailabilityZone,
+						"err", err)
+				}
+
 				labels := model.LabelSet{
 					ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
 				}
@@ -237,6 +274,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 
 				labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId)
 				labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)
+				labels[ec2LabelAZID] = model.LabelValue(azID)
 				labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name)
 				labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType)
 

discovery/azure/azure.go

@@ -46,6 +46,7 @@ const (
 	azureLabelMachineID            = azureLabel + "machine_id"
 	azureLabelMachineResourceGroup = azureLabel + "machine_resource_group"
 	azureLabelMachineName          = azureLabel + "machine_name"
+	azureLabelMachineComputerName  = azureLabel + "machine_computer_name"
 	azureLabelMachineOSType        = azureLabel + "machine_os_type"
 	azureLabelMachineLocation      = azureLabel + "machine_location"
 	azureLabelMachinePrivateIP     = azureLabel + "machine_private_ip"
@@ -226,6 +227,7 @@ type azureResource struct {
 type virtualMachine struct {
 	ID                string
 	Name              string
+	ComputerName      string
 	Type              string
 	Location          string
 	OsType            string
@@ -306,6 +308,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				azureLabelTenantID:             model.LabelValue(d.cfg.TenantID),
 				azureLabelMachineID:            model.LabelValue(vm.ID),
 				azureLabelMachineName:          model.LabelValue(vm.Name),
+				azureLabelMachineComputerName:  model.LabelValue(vm.ComputerName),
 				azureLabelMachineOSType:        model.LabelValue(vm.OsType),
 				azureLabelMachineLocation:      model.LabelValue(vm.Location),
 				azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroup),
@@ -449,6 +452,7 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine {
 	osType := string(vm.StorageProfile.OsDisk.OsType)
 	tags := map[string]*string{}
 	networkInterfaces := []string{}
+	var computerName string
 
 	if vm.Tags != nil {
 		tags = vm.Tags
@@ -460,9 +464,14 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine {
 		}
 	}
 
+	if vm.VirtualMachineProperties != nil && vm.VirtualMachineProperties.OsProfile != nil {
+		computerName = *(vm.VirtualMachineProperties.OsProfile.ComputerName)
+	}
+
 	return virtualMachine{
 		ID:                *(vm.ID),
 		Name:              *(vm.Name),
+		ComputerName:      computerName,
 		Type:              *(vm.Type),
 		Location:          *(vm.Location),
 		OsType:            osType,
@@ -476,6 +485,7 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin
 	osType := string(vm.StorageProfile.OsDisk.OsType)
 	tags := map[string]*string{}
 	networkInterfaces := []string{}
+	var computerName string
 
 	if vm.Tags != nil {
 		tags = vm.Tags
@@ -487,9 +497,14 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin
 		}
 	}
 
+	if vm.VirtualMachineScaleSetVMProperties != nil && vm.VirtualMachineScaleSetVMProperties.OsProfile != nil {
+		computerName = *(vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName)
+	}
+
 	return virtualMachine{
 		ID:                *(vm.ID),
 		Name:              *(vm.Name),
+		ComputerName:      computerName,
 		Type:              *(vm.Type),
 		Location:          *(vm.Location),
 		OsType:            osType,

discovery/azure/azure_test.go

@@ -30,10 +30,14 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 	name := "name"
 	vmType := "type"
 	location := "westeurope"
+	computerName := "computer_name"
 	networkProfile := compute.NetworkProfile{
 		NetworkInterfaces: &[]compute.NetworkInterfaceReference{},
 	}
 	properties := &compute.VirtualMachineProperties{
+		OsProfile: &compute.OSProfile{
+			ComputerName: &computerName,
+		},
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
 				OsType: "Linux",
@@ -54,6 +58,7 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 	expectedVM := virtualMachine{
 		ID:                id,
 		Name:              name,
+		ComputerName:      computerName,
 		Type:              vmType,
 		Location:          location,
 		OsType:            "Linux",
@@ -71,6 +76,7 @@ func TestMapFromVMWithTags(t *testing.T) {
 	name := "name"
 	vmType := "type"
 	location := "westeurope"
+	computerName := "computer_name"
 	tags := map[string]*string{
 		"prometheus": new(string),
 	}
@@ -78,6 +84,9 @@ func TestMapFromVMWithTags(t *testing.T) {
 		NetworkInterfaces: &[]compute.NetworkInterfaceReference{},
 	}
 	properties := &compute.VirtualMachineProperties{
+		OsProfile: &compute.OSProfile{
+			ComputerName: &computerName,
+		},
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
 				OsType: "Linux",
@@ -98,6 +107,7 @@ func TestMapFromVMWithTags(t *testing.T) {
 	expectedVM := virtualMachine{
 		ID:                id,
 		Name:              name,
+		ComputerName:      computerName,
 		Type:              vmType,
 		Location:          location,
 		OsType:            "Linux",
@@ -115,10 +125,14 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 	name := "name"
 	vmType := "type"
 	location := "westeurope"
+	computerName := "computer_name"
 	networkProfile := compute.NetworkProfile{
 		NetworkInterfaces: &[]compute.NetworkInterfaceReference{},
 	}
 	properties := &compute.VirtualMachineScaleSetVMProperties{
+		OsProfile: &compute.OSProfile{
+			ComputerName: &computerName,
+		},
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
 				OsType: "Linux",
@@ -140,6 +154,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 	expectedVM := virtualMachine{
 		ID:                id,
 		Name:              name,
+		ComputerName:      computerName,
 		Type:              vmType,
 		Location:          location,
 		OsType:            "Linux",
@@ -158,6 +173,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 	name := "name"
 	vmType := "type"
 	location := "westeurope"
+	computerName := "computer_name"
 	tags := map[string]*string{
 		"prometheus": new(string),
 	}
@@ -165,6 +181,9 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		NetworkInterfaces: &[]compute.NetworkInterfaceReference{},
 	}
 	properties := &compute.VirtualMachineScaleSetVMProperties{
+		OsProfile: &compute.OSProfile{
+			ComputerName: &computerName,
+		},
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
 				OsType: "Linux",
@@ -186,6 +205,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 	expectedVM := virtualMachine{
 		ID:                id,
 		Name:              name,
+		ComputerName:      computerName,
 		Type:              vmType,
 		Location:          location,
 		OsType:            "Linux",

discovery/consul/consul.go

@@ -199,7 +199,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		logger = log.NewNopLogger()
 	}
 
-	wrapper, err := config.NewClientFromConfig(conf.HTTPClientConfig, "consul_sd", config.WithHTTP2Disabled(), config.WithIdleConnTimeout(2*time.Duration(watchTimeout)))
+	wrapper, err := config.NewClientFromConfig(conf.HTTPClientConfig, "consul_sd", config.WithHTTP2Disabled(), config.WithIdleConnTimeout(2*watchTimeout))
 	if err != nil {
 		return nil, err
 	}

discovery/gce/gce.go

@@ -178,6 +178,12 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 			addr := fmt.Sprintf("%s:%d", priIface.NetworkIP, d.port)
 			labels[model.AddressLabel] = model.LabelValue(addr)
 
+			// Append named interface metadata for all interfaces
+			for _, iface := range inst.NetworkInterfaces {
+				gceLabelNetAddress := model.LabelName(fmt.Sprintf("%sinterface_ipv4_%s", gceLabel, strutil.SanitizeLabelName(iface.Name)))
+				labels[gceLabelNetAddress] = model.LabelValue(iface.NetworkIP)
+			}
+
 			// Tags in GCE are usually only used for networking rules.
 			if inst.Tags != nil && len(inst.Tags.Items) > 0 {
 				// We surround the separated list with the separator as well. This way regular expressions

discovery/hetzner/hcloud.go

@@ -47,6 +47,7 @@ const (
 	hetznerLabelHcloudDiskGB                        = hetznerHcloudLabelPrefix + "disk_size_gb"
 	hetznerLabelHcloudType                          = hetznerHcloudLabelPrefix + "server_type"
 	hetznerLabelHcloudLabel                         = hetznerHcloudLabelPrefix + "label_"
+	hetznerLabelHcloudLabelPresent                  = hetznerHcloudLabelPrefix + "labelpresent_"
 )
 
 // Discovery periodically performs Hetzner Cloud requests. It implements
@@ -124,6 +125,9 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 			}
 		}
 		for labelKey, labelValue := range server.Labels {
+			presentLabel := model.LabelName(hetznerLabelHcloudLabelPresent + strutil.SanitizeLabelName(labelKey))
+			labels[presentLabel] = model.LabelValue("true")
+
 			label := model.LabelName(hetznerLabelHcloudLabel + strutil.SanitizeLabelName(labelKey))
 			labels[label] = model.LabelValue(labelValue)
 		}

discovery/hetzner/hcloud_test.go

@@ -77,6 +77,7 @@ func TestHCloudSDRefresh(t *testing.T) {
 			"__meta_hetzner_hcloud_disk_size_gb":                     model.LabelValue("25"),
 			"__meta_hetzner_hcloud_server_type":                      model.LabelValue("cx11"),
 			"__meta_hetzner_hcloud_private_ipv4_mynet":               model.LabelValue("10.0.0.2"),
+			"__meta_hetzner_hcloud_labelpresent_my_key":              model.LabelValue("true"),
 			"__meta_hetzner_hcloud_label_my_key":                     model.LabelValue("my-value"),
 		},
 		{
@@ -99,6 +100,10 @@ func TestHCloudSDRefresh(t *testing.T) {
 			"__meta_hetzner_hcloud_memory_size_gb":                   model.LabelValue("1"),
 			"__meta_hetzner_hcloud_disk_size_gb":                     model.LabelValue("50"),
 			"__meta_hetzner_hcloud_server_type":                      model.LabelValue("cpx11"),
+			"__meta_hetzner_hcloud_labelpresent_key":                 model.LabelValue("true"),
+			"__meta_hetzner_hcloud_labelpresent_other_key":           model.LabelValue("true"),
+			"__meta_hetzner_hcloud_label_key":                        model.LabelValue(""),
+			"__meta_hetzner_hcloud_label_other_key":                  model.LabelValue("value"),
 		},
 		{
 			"__address__":                                            model.LabelValue("1.2.3.6:80"),

discovery/hetzner/mock_test.go

@@ -310,7 +310,10 @@ func (m *SDMock) HandleHcloudServers() {
         "delete": false,
         "rebuild": false
       },
-      "labels": {},
+      "labels": {
+        "key": "",
+        "other-key": "value"
+      },
       "volumes": [],
       "load_balancers": []
     },

discovery/hetzner/robot.go

@@ -92,8 +92,13 @@ func (d *robotDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, err
 		return nil, errors.Errorf("non 2xx status '%d' response during hetzner service discovery with role robot", resp.StatusCode)
 	}
 
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
 	var servers serversList
-	err = json.NewDecoder(resp.Body).Decode(&servers)
+	err = json.Unmarshal(b, &servers)
 	if err != nil {
 		return nil, err
 	}

discovery/http/http.go

@@ -21,7 +21,9 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-kit/log"
@@ -41,7 +43,8 @@ var (
 		RefreshInterval:  model.Duration(60 * time.Second),
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
 	}
-	userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
+	userAgent        = fmt.Sprintf("Prometheus/%s", version.Version)
+	matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
 )
 
 func init() {
@@ -101,6 +104,7 @@ type Discovery struct {
 	url             string
 	client          *http.Client
 	refreshInterval time.Duration
+	tgLastLength    int
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
@@ -152,7 +156,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 		return nil, errors.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
-	if resp.Header.Get("Content-Type") != "application/json" {
+	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
 		return nil, errors.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
@@ -180,6 +184,13 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 		tg.Labels[httpSDURLLabel] = model.LabelValue(d.url)
 	}
 
+	// Generate empty updates for sources that disappeared.
+	l := len(targetGroups)
+	for i := l; i < d.tgLastLength; i++ {
+		targetGroups = append(targetGroups, &targetgroup.Group{Source: urlSource(d.url, i)})
+	}
+	d.tgLastLength = l
+
 	return targetGroups, nil
 }
 

discovery/http/http_test.go

@@ -104,3 +104,299 @@ func TestHTTPInvalidFormat(t *testing.T) {
 	_, err = d.refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
 }
+
+func TestContentTypeRegex(t *testing.T) {
+	cases := []struct {
+		header string
+		match  bool
+	}{
+		{
+			header: "application/json;charset=utf-8",
+			match:  true,
+		},
+		{
+			header: "application/json;charset=UTF-8",
+			match:  true,
+		},
+		{
+			header: "Application/JSON;Charset=\"utf-8\"",
+			match:  true,
+		},
+		{
+			header: "application/json; charset=\"utf-8\"",
+			match:  true,
+		},
+		{
+			header: "application/json",
+			match:  true,
+		},
+		{
+			header: "application/jsonl; charset=\"utf-8\"",
+			match:  false,
+		},
+		{
+			header: "application/json;charset=UTF-9",
+			match:  false,
+		},
+		{
+			header: "application /json;charset=UTF-8",
+			match:  false,
+		},
+		{
+			header: "application/ json;charset=UTF-8",
+			match:  false,
+		},
+		{
+			header: "application/json;",
+			match:  false,
+		},
+		{
+			header: "charset=UTF-8",
+			match:  false,
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.header, func(t *testing.T) {
+			require.Equal(t, test.match, matchContentType.MatchString(test.header))
+		})
+	}
+}
+
+func TestSourceDisappeared(t *testing.T) {
+	var stubResponse string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintln(w, stubResponse)
+	}))
+	t.Cleanup(ts.Close)
+
+	cases := []struct {
+		responses       []string
+		expectedTargets [][]*targetgroup.Group
+	}{
+		{
+			responses: []string{
+				`[]`,
+				`[]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{{}, {}},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "3"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "v"}, "targets": ["127.0.0.2"]}, {"labels": {"k": "vv"}, "targets": ["127.0.0.3"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("3"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 2),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 1),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 2),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.2"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("v"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.3"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("vv"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+	}
+
+	cfg := SDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		URL:              ts.URL,
+		RefreshInterval:  model.Duration(1 * time.Second),
+	}
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+	for _, test := range cases {
+		ctx := context.Background()
+		for i, res := range test.responses {
+			stubResponse = res
+			tgs, err := d.refresh(ctx)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedTargets[i], tgs)
+		}
+	}
+
+}

discovery/install/install.go

@@ -33,5 +33,6 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/openstack"    // register openstack
 	_ "github.com/prometheus/prometheus/discovery/scaleway"     // register scaleway
 	_ "github.com/prometheus/prometheus/discovery/triton"       // register triton
+	_ "github.com/prometheus/prometheus/discovery/xds"          // register xds
 	_ "github.com/prometheus/prometheus/discovery/zookeeper"    // register zookeeper
 )

discovery/linode/linode.go

@@ -15,6 +15,7 @@ package linode
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -56,6 +57,11 @@ const (
 	linodeLabelSpecsVCPUs         = linodeLabel + "specs_vcpus"
 	linodeLabelSpecsTransferBytes = linodeLabel + "specs_transfer_bytes"
 	linodeLabelExtraIPs           = linodeLabel + "extra_ips"
+
+	// This is our events filter; when polling for changes, we care only about
+	// events since our last refresh.
+	// Docs: https://www.linode.com/docs/api/account/#events-list
+	filterTemplate = `{"created": {"+gte": "%s"}}`
 )
 
 // DefaultSDConfig is the default Linode SD configuration.
@@ -107,16 +113,23 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	client       *linodego.Client
+	client               *linodego.Client
-	port         int
+	port                 int
-	tagSeparator string
+	tagSeparator         string
+	lastRefreshTimestamp time.Time
+	pollCount            int
+	lastResults          []*targetgroup.Group
+	eventPollingEnabled  bool
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
 func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d := &Discovery{
-		port:         conf.Port,
+		port:                 conf.Port,
-		tagSeparator: conf.TagSeparator,
+		tagSeparator:         conf.TagSeparator,
+		pollCount:            0,
+		lastRefreshTimestamp: time.Now().UTC(),
+		eventPollingEnabled:  true,
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd", config.WithHTTP2Disabled())
@@ -143,6 +156,50 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 }
 
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	needsRefresh := true
+	ts := time.Now().UTC()
+
+	if d.lastResults != nil && d.eventPollingEnabled {
+		// Check to see if there have been any events. If so, refresh our data.
+		opts := linodego.NewListOptions(1, fmt.Sprintf(filterTemplate, d.lastRefreshTimestamp.Format("2006-01-02T15:04:05")))
+		events, err := d.client.ListEvents(ctx, opts)
+		if err != nil {
+			var e *linodego.Error
+			if errors.As(err, &e) && e.Code == http.StatusUnauthorized {
+				// If we get a 401, the token doesn't have `events:read_only` scope.
+				// Disable event polling and fallback to doing a full refresh every interval.
+				d.eventPollingEnabled = false
+			} else {
+				return nil, err
+			}
+		} else {
+			// Event polling tells us changes the Linode API is aware of. Actions issued outside of the Linode API,
+			// such as issuing a `shutdown` at the VM's console instead of using the API to power off an instance,
+			// can potentially cause us to return stale data. Just in case, trigger a full refresh after ~10 polling
+			// intervals of no events.
+			d.pollCount++
+
+			if len(events) == 0 && d.pollCount < 10 {
+				needsRefresh = false
+			}
+		}
+	}
+
+	if needsRefresh {
+		newData, err := d.refreshData(ctx)
+		if err != nil {
+			return nil, err
+		}
+		d.pollCount = 0
+		d.lastResults = newData
+	}
+
+	d.lastRefreshTimestamp = ts
+
+	return d.lastResults, nil
+}
+
+func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, error) {
 	tg := &targetgroup.Group{
 		Source: "Linode",
 	}

discovery/linode/linode_test.go

@@ -39,6 +39,7 @@ func (s *LinodeSDTestSuite) SetupTest(t *testing.T) {
 
 	s.Mock.HandleLinodeInstancesList()
 	s.Mock.HandleLinodeNeworkingIPs()
+	s.Mock.HandleLinodeAccountEvents()
 }
 
 func TestLinodeSDRefresh(t *testing.T) {

discovery/linode/mock_test.go

@@ -413,3 +413,42 @@ func (m *SDMock) HandleLinodeNeworkingIPs() {
 		)
 	})
 }
+
+// HandleLinodeAccountEvents mocks linode the account/events endpoint.
+func (m *SDMock) HandleLinodeAccountEvents() {
+	m.Mux.HandleFunc("/v4/account/events", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+
+		if r.Header.Get("X-Filter") == "" {
+			// This should never happen; if the client sends an events request without
+			// a filter, cause it to fail. The error below is not a real response from
+			// the API, but should aid in debugging failed tests.
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, `
+{
+	"errors": [
+		{
+			"reason": "Request missing expected X-Filter headers"
+		}
+	]
+}`,
+			)
+			return
+		}
+
+		w.Header().Set("content-type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+
+		fmt.Fprint(w, `
+{
+	"data": [],
+	"results": 0,
+	"pages": 1,
+	"page": 1
+}`,
+		)
+	})
+}

discovery/marathon/marathon.go

@@ -339,8 +339,13 @@ func fetchApps(ctx context.Context, client *http.Client, url string) (*appList,
 		return nil, errors.Errorf("non 2xx status '%v' response during marathon service discovery", resp.StatusCode)
 	}
 
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
 	var apps appList
-	err = json.NewDecoder(resp.Body).Decode(&apps)
+	err = json.Unmarshal(b, &apps)
 	if err != nil {
 		return nil, errors.Wrapf(err, "%q", url)
 	}

discovery/moby/docker.go

@@ -51,10 +51,11 @@ const (
 
 // DefaultDockerSDConfig is the default Docker SD configuration.
 var DefaultDockerSDConfig = DockerSDConfig{
-	RefreshInterval:  model.Duration(60 * time.Second),
+	RefreshInterval:    model.Duration(60 * time.Second),
-	Port:             80,
+	Port:               80,
-	Filters:          []Filter{},
+	Filters:            []Filter{},
-	HTTPClientConfig: config.DefaultHTTPClientConfig,
+	HostNetworkingHost: "localhost",
+	HTTPClientConfig:   config.DefaultHTTPClientConfig,
 }
 
 func init() {
@@ -65,9 +66,10 @@ func init() {
 type DockerSDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 
-	Host    string   `yaml:"host"`
+	Host               string   `yaml:"host"`
-	Port    int      `yaml:"port"`
+	Port               int      `yaml:"port"`
-	Filters []Filter `yaml:"filters"`
+	Filters            []Filter `yaml:"filters"`
+	HostNetworkingHost string   `yaml:"host_networking_host"`
 
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 }
@@ -104,9 +106,10 @@ func (c *DockerSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
 
 type DockerDiscovery struct {
 	*refresh.Discovery
-	client  *client.Client
+	client             *client.Client
-	port    int
+	port               int
-	filters filters.Args
+	hostNetworkingHost string
+	filters            filters.Args
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
@@ -114,7 +117,8 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscove
 	var err error
 
 	d := &DockerDiscovery{
-		port: conf.Port,
+		port:               conf.Port,
+		hostNetworkingHost: conf.HostNetworkingHost,
 	}
 
 	hostURL, err := url.Parse(conf.Host)
@@ -245,7 +249,15 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 					labels[model.LabelName(k)] = model.LabelValue(v)
 				}
 
-				addr := net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(d.port), 10))
+				// Containers in host networking mode don't have ports,
+				// so they only end up here, not in the previous loop.
+				var addr string
+				if c.HostConfig.NetworkMode != "host" {
+					addr = net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(d.port), 10))
+				} else {
+					addr = d.hostNetworkingHost
+				}
+
 				labels[model.AddressLabel] = model.LabelValue(addr)
 				tg.Targets = append(tg.Targets, labels)
 			}

discovery/moby/docker_test.go

@@ -49,7 +49,7 @@ host: %s
 	tg := tgs[0]
 	require.NotNil(t, tg)
 	require.NotNil(t, tg.Targets)
-	require.Equal(t, 2, len(tg.Targets))
+	require.Equal(t, 3, len(tg.Targets))
 
 	for i, lbls := range []model.LabelSet{
 		{
@@ -93,6 +93,16 @@ host: %s
 			"__meta_docker_network_name":                               "dockersd_default",
 			"__meta_docker_network_scope":                              "local",
 		},
+		{
+			"__address__":                "localhost",
+			"__meta_docker_container_id": "54ed6cc5c0988260436cb0e739b7b6c9cad6c439a93b4c4fdbe9753e1c94b189",
+			"__meta_docker_container_label_com_docker_compose_project": "dockersd",
+			"__meta_docker_container_label_com_docker_compose_service": "host_networking",
+			"__meta_docker_container_label_com_docker_compose_version": "1.25.0",
+			"__meta_docker_container_name":                             "/dockersd_host_networking_1",
+			"__meta_docker_container_network_mode":                     "host",
+			"__meta_docker_network_ip":                                 "",
+		},
 	} {
 		t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
 			require.Equal(t, lbls, tg.Targets[i])

discovery/moby/testdata/dockerprom/containers/json_limit_0.json

@@ -89,5 +89,44 @@
       }
     },
     "Mounts": []
+  },
+  {
+    "Id": "54ed6cc5c0988260436cb0e739b7b6c9cad6c439a93b4c4fdbe9753e1c94b189",
+    "Names": [
+      "/dockersd_host_networking_1"
+    ],
+    "Image": "httpd",
+    "ImageID": "sha256:73b8cfec11558fe86f565b4357f6d6c8560f4c49a5f15ae970a24da86c9adc93",
+    "Command": "httpd-foreground",
+    "Created": 1627440494,
+    "Ports": [],
+    "Labels": {
+      "com.docker.compose.project": "dockersd",
+      "com.docker.compose.service": "host_networking",
+      "com.docker.compose.version": "1.25.0"
+    },
+    "State": "running",
+    "Status": "Up 3 minutes",
+    "HostConfig": { "NetworkMode": "host" },
+    "NetworkSettings": {
+      "Networks": {
+        "host": {
+          "IPAMConfig": null,
+          "Links": null,
+          "Aliases": null,
+          "NetworkID": "80c4459fa193c5c8b57e90e117d2f899d1a86708e548738149d62e03df0ec35c",
+          "EndpointID": "e9bea4c499c34bd41609b0e1e9b38f9964c69180c1a22130f28b6af802c156d8",
+          "Gateway": "",
+          "IPAddress": "",
+          "IPPrefixLen": 0,
+          "IPv6Gateway": "",
+          "GlobalIPv6Address": "",
+          "GlobalIPv6PrefixLen": 0,
+          "MacAddress": "",
+          "DriverOpts": null
+        }
+      }
+    },
+    "Mounts": []
   }
 ]

discovery/openstack/hypervisor.go

@@ -74,7 +74,7 @@ func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group
 	}
 	// OpenStack API reference
 	// https://developer.openstack.org/api-ref/compute/#list-hypervisors-details
-	pagerHypervisors := hypervisors.List(client)
+	pagerHypervisors := hypervisors.List(client, nil)
 	err = pagerHypervisors.EachPage(func(page pagination.Page) (bool, error) {
 		hypervisorList, err := hypervisors.ExtractHypervisors(page)
 		if err != nil {

discovery/xds/client.go

@@ -0,0 +1,226 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	envoy_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/version"
+)
+
+var userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
+
+// ResourceClient exposes the xDS protocol for a single resource type.
+// See https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#rest-json-polling-subscriptions .
+type ResourceClient interface {
+	// ResourceTypeURL is the type URL of the resource.
+	ResourceTypeURL() string
+
+	// Server is the xDS Management server.
+	Server() string
+
+	// Fetch requests the latest view of the entire resource state.
+	// If no updates have been made since the last request, the response will be nil.
+	Fetch(ctx context.Context) (*v3.DiscoveryResponse, error)
+
+	// ID returns the ID of the client that is sent to the xDS server.
+	ID() string
+
+	// Close releases currently held resources.
+	Close()
+}
+
+type HTTPResourceClient struct {
+	client *http.Client
+	config *HTTPResourceClientConfig
+
+	// endpoint is the fully-constructed xDS HTTP endpoint.
+	endpoint string
+	// Caching.
+	latestVersion string
+	latestNonce   string
+}
+
+type HTTPResourceClientConfig struct {
+	// HTTP config.
+	config.HTTPClientConfig
+
+	Name string
+
+	// ExtraQueryParams are extra query parameters to attach to the request URL.
+	ExtraQueryParams url.Values
+
+	// General xDS config.
+
+	// The timeout for a single fetch request.
+	Timeout time.Duration
+
+	// Type is the xds type, e.g., clusters
+	// which is used in the discovery POST request.
+	ResourceType string
+	// ResourceTypeURL is the Google type url for the resource, e.g., type.googleapis.com/envoy.api.v2.Cluster.
+	ResourceTypeURL string
+	// Server is the xDS management server.
+	Server string
+	// ClientID is used to identify the client with the management server.
+	ClientID string
+}
+
+func NewHTTPResourceClient(conf *HTTPResourceClientConfig, protocolVersion ProtocolVersion) (*HTTPResourceClient, error) {
+	if protocolVersion != ProtocolV3 {
+		return nil, errors.New("only the v3 protocol is supported")
+	}
+
+	if len(conf.Server) == 0 {
+		return nil, errors.New("empty xDS server")
+	}
+
+	serverURL, err := url.Parse(conf.Server)
+	if err != nil {
+		return nil, err
+	}
+
+	endpointURL, err := makeXDSResourceHTTPEndpointURL(protocolVersion, serverURL, conf.ResourceType)
+	if err != nil {
+		return nil, err
+	}
+
+	if conf.ExtraQueryParams != nil {
+		endpointURL.RawQuery = conf.ExtraQueryParams.Encode()
+	}
+
+	client, err := config.NewClientFromConfig(conf.HTTPClientConfig, conf.Name, config.WithHTTP2Disabled(), config.WithIdleConnTimeout(conf.Timeout))
+	if err != nil {
+		return nil, err
+	}
+
+	client.Timeout = conf.Timeout
+
+	return &HTTPResourceClient{
+		client:        client,
+		config:        conf,
+		endpoint:      endpointURL.String(),
+		latestVersion: "",
+		latestNonce:   "",
+	}, nil
+}
+
+func makeXDSResourceHTTPEndpointURL(protocolVersion ProtocolVersion, serverURL *url.URL, resourceType string) (*url.URL, error) {
+	if serverURL == nil {
+		return nil, errors.New("empty xDS server URL")
+	}
+
+	if len(serverURL.Scheme) == 0 || len(serverURL.Host) == 0 {
+		return nil, errors.New("invalid xDS server URL")
+	}
+
+	if serverURL.Scheme != "http" && serverURL.Scheme != "https" {
+		return nil, errors.New("invalid xDS server URL protocol. must be either 'http' or 'https'")
+	}
+
+	serverURL.Path = path.Join(serverURL.Path, string(protocolVersion), fmt.Sprintf("discovery:%s", resourceType))
+
+	return serverURL, nil
+}
+
+func (rc *HTTPResourceClient) Server() string {
+	return rc.config.Server
+}
+
+func (rc *HTTPResourceClient) ResourceTypeURL() string {
+	return rc.config.ResourceTypeURL
+}
+
+func (rc *HTTPResourceClient) ID() string {
+	return rc.config.ClientID
+}
+
+func (rc *HTTPResourceClient) Close() {
+	rc.client.CloseIdleConnections()
+}
+
+// Fetch requests the latest state of the resources from the xDS server and cache the version.
+// Returns a nil response if the current local version is up to date.
+func (rc *HTTPResourceClient) Fetch(ctx context.Context) (*v3.DiscoveryResponse, error) {
+	discoveryReq := &v3.DiscoveryRequest{
+		VersionInfo:   rc.latestVersion,
+		ResponseNonce: rc.latestNonce,
+		TypeUrl:       rc.ResourceTypeURL(),
+		ResourceNames: []string{},
+		Node: &envoy_core.Node{
+			Id: rc.ID(),
+		},
+	}
+
+	reqBody, err := protoJSONMarshalOptions.Marshal(discoveryReq)
+	if err != nil {
+		return nil, err
+	}
+
+	request, err := http.NewRequest("POST", rc.endpoint, bytes.NewBuffer(reqBody))
+	if err != nil {
+		return nil, err
+	}
+	request = request.WithContext(ctx)
+
+	request.Header.Add("User-Agent", userAgent)
+	request.Header.Add("Content-Type", "application/json")
+	request.Header.Add("Accept", "application/json")
+
+	resp, err := rc.client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	if resp.StatusCode == http.StatusNotModified {
+		// Empty response, already have the latest.
+		return nil, nil
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("non 200 status '%d' response during xDS fetch", resp.StatusCode)
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	discoveryRes := &v3.DiscoveryResponse{}
+	if err = protoJSONUnmarshalOptions.Unmarshal(respBody, discoveryRes); err != nil {
+		return nil, err
+	}
+
+	// Cache the latest nonce + version info.
+	rc.latestNonce = discoveryRes.Nonce
+	rc.latestVersion = discoveryRes.VersionInfo
+
+	return discoveryRes, nil
+}

discovery/xds/client_test.go

@@ -0,0 +1,161 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"testing"
+	"time"
+
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/prometheus/common/config"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/anypb"
+)
+
+var (
+	httpResourceConf = &HTTPResourceClientConfig{
+		HTTPClientConfig: config.HTTPClientConfig{
+			TLSConfig: config.TLSConfig{InsecureSkipVerify: true},
+		},
+		ResourceType: "monitoring",
+		// Some known type.
+		ResourceTypeURL: "type.googleapis.com/envoy.service.discovery.v3.DiscoveryRequest",
+		Server:          "http://localhost",
+		ClientID:        "test-id",
+	}
+)
+
+func urlMustParse(str string) *url.URL {
+	parsed, err := url.Parse(str)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return parsed
+}
+
+func TestMakeXDSResourceHttpEndpointEmptyServerURLScheme(t *testing.T) {
+	endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("127.0.0.1"), "monitoring")
+
+	require.Empty(t, endpointURL)
+	require.Error(t, err)
+	require.Equal(t, err.Error(), "invalid xDS server URL")
+}
+
+func TestMakeXDSResourceHttpEndpointEmptyServerURLHost(t *testing.T) {
+	endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("grpc://127.0.0.1"), "monitoring")
+
+	require.Empty(t, endpointURL)
+	require.NotNil(t, err)
+	require.Contains(t, err.Error(), "must be either 'http' or 'https'")
+}
+
+func TestMakeXDSResourceHttpEndpoint(t *testing.T) {
+	endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("http://127.0.0.1:5000"), "monitoring")
+
+	require.NoError(t, err)
+	require.Equal(t, endpointURL.String(), "http://127.0.0.1:5000/v3/discovery:monitoring")
+}
+
+func TestCreateNewHTTPResourceClient(t *testing.T) {
+	c := &HTTPResourceClientConfig{
+		HTTPClientConfig: sdConf.HTTPClientConfig,
+		Name:             "test",
+		ExtraQueryParams: url.Values{
+			"param1": {"v1"},
+		},
+		Timeout:         1 * time.Minute,
+		ResourceType:    "monitoring",
+		ResourceTypeURL: "type.googleapis.com/envoy.service.discovery.v3.DiscoveryRequest",
+		Server:          "http://127.0.0.1:5000",
+		ClientID:        "client",
+	}
+
+	client, err := NewHTTPResourceClient(c, ProtocolV3)
+
+	require.NoError(t, err)
+
+	require.Equal(t, client.endpoint, "http://127.0.0.1:5000/v3/discovery:monitoring?param1=v1")
+	require.Equal(t, client.client.Timeout, 1*time.Minute)
+
+}
+
+func createTestHTTPResourceClient(t *testing.T, conf *HTTPResourceClientConfig, protocolVersion ProtocolVersion, responder discoveryResponder) (*HTTPResourceClient, func()) {
+	s := createTestHTTPServer(t, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
+		require.Equal(t, conf.ResourceTypeURL, request.TypeUrl)
+		require.Equal(t, conf.ClientID, request.Node.Id)
+		return responder(request)
+	})
+
+	conf.Server = s.URL
+	client, err := NewHTTPResourceClient(conf, protocolVersion)
+	require.NoError(t, err)
+
+	return client, s.Close
+}
+
+func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) {
+	client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
+		return nil, nil
+	})
+	defer cleanup()
+
+	res, err := client.Fetch(context.Background())
+	require.Nil(t, res)
+	require.NoError(t, err)
+}
+
+func TestHTTPResourceClientFetchFullResponse(t *testing.T) {
+	client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
+		if request.VersionInfo == "1" {
+			return nil, nil
+		}
+
+		return &v3.DiscoveryResponse{
+			TypeUrl:     request.TypeUrl,
+			VersionInfo: "1",
+			Nonce:       "abc",
+			Resources:   []*anypb.Any{},
+		}, nil
+	})
+	defer cleanup()
+
+	res, err := client.Fetch(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, res)
+
+	require.Equal(t, client.ResourceTypeURL(), res.TypeUrl)
+	require.Len(t, res.Resources, 0)
+	require.Equal(t, "abc", client.latestNonce, "Nonce not cached")
+	require.Equal(t, "1", client.latestVersion, "Version not cached")
+
+	res, err = client.Fetch(context.Background())
+	require.Nil(t, res, "Update not expected")
+	require.NoError(t, err)
+}
+
+func TestHTTPResourceClientServerError(t *testing.T) {
+	client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
+		return nil, errors.New("server error")
+	})
+	defer cleanup()
+
+	res, err := client.Fetch(context.Background())
+	require.Nil(t, res)
+	require.Error(t, err)
+}

discovery/xds/kuma.go

@@ -0,0 +1,226 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"github.com/prometheus/prometheus/util/osutil"
+	"github.com/prometheus/prometheus/util/strutil"
+)
+
+var (
+	// DefaultKumaSDConfig is the default Kuma MADS SD configuration.
+	DefaultKumaSDConfig = KumaSDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		RefreshInterval:  model.Duration(15 * time.Second),
+		FetchTimeout:     model.Duration(2 * time.Minute),
+	}
+
+	kumaFetchFailuresCount = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "sd_kuma_fetch_failures_total",
+			Help:      "The number of Kuma MADS fetch call failures.",
+		})
+	kumaFetchSkipUpdateCount = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "sd_kuma_fetch_skipped_updates_total",
+			Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
+		})
+	kumaFetchDuration = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Namespace:  namespace,
+			Name:       "sd_kuma_fetch_duration_seconds",
+			Help:       "The duration of a Kuma MADS fetch call.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+	)
+)
+
+const (
+	// kumaMetaLabelPrefix is the meta prefix used for all kuma meta labels.
+	kumaMetaLabelPrefix = model.MetaLabelPrefix + "kuma_"
+
+	// kumaMeshLabel is the name of the label that holds the mesh name.
+	kumaMeshLabel = kumaMetaLabelPrefix + "mesh"
+	// kumaServiceLabel is the name of the label that holds the service name.
+	kumaServiceLabel = kumaMetaLabelPrefix + "service"
+	// kumaDataplaneLabel is the name of the label that holds the dataplane name.
+	kumaDataplaneLabel = kumaMetaLabelPrefix + "dataplane"
+	// kumaUserLabelPrefix is the name of the label that namespaces all user-defined labels.
+	kumaUserLabelPrefix = kumaMetaLabelPrefix + "label_"
+)
+
+const (
+	KumaMadsV1ResourceTypeURL = "type.googleapis.com/kuma.observability.v1.MonitoringAssignment"
+	KumaMadsV1ResourceType    = "monitoringassignments"
+)
+
+type KumaSDConfig = SDConfig
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultKumaSDConfig
+	type plainKumaConf KumaSDConfig
+	err := unmarshal((*plainKumaConf)(c))
+	if err != nil {
+		return err
+	}
+
+	if len(c.Server) == 0 {
+		return errors.Errorf("kuma SD server must not be empty: %s", c.Server)
+	}
+	parsedURL, err := url.Parse(c.Server)
+	if err != nil {
+		return err
+	}
+
+	if len(parsedURL.Scheme) == 0 || len(parsedURL.Host) == 0 {
+		return errors.Errorf("kuma SD server must not be empty and have a scheme: %s", c.Server)
+	}
+
+	if err := c.HTTPClientConfig.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *KumaSDConfig) Name() string {
+	return "kuma"
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *KumaSDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
+func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	logger := opts.Logger
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
+
+	return NewKumaHTTPDiscovery(c, logger)
+}
+
+func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) *targetgroup.Group {
+	commonLabels := convertKumaUserLabels(assignment.Labels)
+
+	commonLabels[kumaMeshLabel] = model.LabelValue(assignment.Mesh)
+	commonLabels[kumaServiceLabel] = model.LabelValue(assignment.Service)
+
+	var targetLabelSets []model.LabelSet
+
+	for _, target := range assignment.Targets {
+		targetLabels := convertKumaUserLabels(target.Labels)
+
+		targetLabels[kumaDataplaneLabel] = model.LabelValue(target.Name)
+		targetLabels[model.InstanceLabel] = model.LabelValue(target.Name)
+		targetLabels[model.AddressLabel] = model.LabelValue(target.Address)
+		targetLabels[model.SchemeLabel] = model.LabelValue(target.Scheme)
+		targetLabels[model.MetricsPathLabel] = model.LabelValue(target.MetricsPath)
+
+		targetLabelSets = append(targetLabelSets, targetLabels)
+	}
+
+	return &targetgroup.Group{
+		Labels:  commonLabels,
+		Targets: targetLabelSets,
+	}
+}
+
+func convertKumaUserLabels(labels map[string]string) model.LabelSet {
+	labelSet := model.LabelSet{}
+	for key, value := range labels {
+		name := kumaUserLabelPrefix + strutil.SanitizeLabelName(key)
+		labelSet[model.LabelName(name)] = model.LabelValue(value)
+	}
+	return labelSet
+}
+
+// kumaMadsV1ResourceParser is an xds.resourceParser.
+func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]*targetgroup.Group, error) {
+	if typeURL != KumaMadsV1ResourceTypeURL {
+		return nil, errors.Errorf("recieved invalid typeURL for Kuma MADS v1 Resource: %s", typeURL)
+	}
+
+	var groups []*targetgroup.Group
+
+	for _, resource := range resources {
+		assignment := &MonitoringAssignment{}
+
+		if err := anypb.UnmarshalTo(resource, assignment, protoUnmarshalOptions); err != nil {
+			return nil, err
+		}
+
+		groups = append(groups, convertKumaV1MonitoringAssignment(assignment))
+	}
+
+	return groups, nil
+}
+
+func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
+	// Default to "prometheus" if hostname is unavailable.
+	clientID, err := osutil.GetFQDN()
+	if err != nil {
+		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
+		clientID = "prometheus"
+	}
+
+	clientConfig := &HTTPResourceClientConfig{
+		HTTPClientConfig: conf.HTTPClientConfig,
+		ExtraQueryParams: url.Values{
+			"fetch-timeout": {conf.FetchTimeout.String()},
+		},
+		// Allow 15s of buffer over the timeout sent to the xDS server for connection overhead.
+		Timeout:         time.Duration(conf.FetchTimeout) + (15 * time.Second),
+		ResourceType:    KumaMadsV1ResourceType,
+		ResourceTypeURL: KumaMadsV1ResourceTypeURL,
+		Server:          conf.Server,
+		ClientID:        clientID,
+	}
+
+	client, err := NewHTTPResourceClient(clientConfig, ProtocolV3)
+	if err != nil {
+		return nil, fmt.Errorf("kuma_sd: %w", err)
+	}
+
+	d := &fetchDiscovery{
+		client:               client,
+		logger:               logger,
+		refreshInterval:      time.Duration(conf.RefreshInterval),
+		source:               "kuma",
+		parseResources:       kumaMadsV1ResourceParser,
+		fetchFailuresCount:   kumaFetchFailuresCount,
+		fetchSkipUpdateCount: kumaFetchSkipUpdateCount,
+		fetchDuration:        kumaFetchDuration,
+	}
+
+	return d, nil
+}

discovery/xds/kuma_mads.pb.go

@@ -0,0 +1,398 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: observability/v1/mads.proto
+
+// gRPC-removed vendored file from Kuma.
+
+package xds
+
+import (
+	context "context"
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	_ "github.com/envoyproxy/protoc-gen-validate/validate"
+	_ "google.golang.org/genproto/googleapis/api/annotations"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// MADS resource type.
+//
+// Describes a group of targets on a single service that need to be monitored.
+type MonitoringAssignment struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Mesh of the dataplane.
+	//
+	// E.g., `default`
+	Mesh string `protobuf:"bytes,2,opt,name=mesh,proto3" json:"mesh,omitempty"`
+	// Identifying service the dataplane is proxying.
+	//
+	// E.g., `backend`
+	Service string `protobuf:"bytes,3,opt,name=service,proto3" json:"service,omitempty"`
+	// List of targets that need to be monitored.
+	Targets []*MonitoringAssignment_Target `protobuf:"bytes,4,rep,name=targets,proto3" json:"targets,omitempty"`
+	// Arbitrary Labels associated with every target in the assignment.
+	//
+	// E.g., `{"zone" : "us-east-1", "team": "infra", "commit_hash": "620506a88"}`.
+	Labels map[string]string `protobuf:"bytes,5,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *MonitoringAssignment) Reset() {
+	*x = MonitoringAssignment{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_observability_v1_mads_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *MonitoringAssignment) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MonitoringAssignment) ProtoMessage() {}
+
+func (x *MonitoringAssignment) ProtoReflect() protoreflect.Message {
+	mi := &file_observability_v1_mads_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MonitoringAssignment.ProtoReflect.Descriptor instead.
+func (*MonitoringAssignment) Descriptor() ([]byte, []int) {
+	return file_observability_v1_mads_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *MonitoringAssignment) GetMesh() string {
+	if x != nil {
+		return x.Mesh
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment) GetTargets() []*MonitoringAssignment_Target {
+	if x != nil {
+		return x.Targets
+	}
+	return nil
+}
+
+func (x *MonitoringAssignment) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+// Describes a single target that needs to be monitored.
+type MonitoringAssignment_Target struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// E.g., `backend-01`
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// Scheme on which to scrape the target.
+	//E.g., `http`
+	Scheme string `protobuf:"bytes,2,opt,name=scheme,proto3" json:"scheme,omitempty"`
+	// Address (preferably IP) for the service
+	// E.g., `backend.svc` or `10.1.4.32:9090`
+	Address string `protobuf:"bytes,3,opt,name=address,proto3" json:"address,omitempty"`
+	// Optional path to append to the address for scraping
+	//E.g., `/metrics`
+	MetricsPath string `protobuf:"bytes,4,opt,name=metrics_path,json=metricsPath,proto3" json:"metrics_path,omitempty"`
+	// Arbitrary labels associated with that particular target.
+	//
+	// E.g.,
+	// `{
+	//    "commit_hash" : "620506a88",
+	//  }`.
+	Labels map[string]string `protobuf:"bytes,5,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *MonitoringAssignment_Target) Reset() {
+	*x = MonitoringAssignment_Target{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_observability_v1_mads_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *MonitoringAssignment_Target) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MonitoringAssignment_Target) ProtoMessage() {}
+
+func (x *MonitoringAssignment_Target) ProtoReflect() protoreflect.Message {
+	mi := &file_observability_v1_mads_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MonitoringAssignment_Target.ProtoReflect.Descriptor instead.
+func (*MonitoringAssignment_Target) Descriptor() ([]byte, []int) {
+	return file_observability_v1_mads_proto_rawDescGZIP(), []int{0, 0}
+}
+
+func (x *MonitoringAssignment_Target) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment_Target) GetScheme() string {
+	if x != nil {
+		return x.Scheme
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment_Target) GetAddress() string {
+	if x != nil {
+		return x.Address
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment_Target) GetMetricsPath() string {
+	if x != nil {
+		return x.MetricsPath
+	}
+	return ""
+}
+
+func (x *MonitoringAssignment_Target) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+var File_observability_v1_mads_proto protoreflect.FileDescriptor
+
+var file_observability_v1_mads_proto_rawDesc = []byte{
+	0x0a, 0x1b, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x2f,
+	0x76, 0x31, 0x2f, 0x6d, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x15, 0x6b,
+	0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74,
+	0x79, 0x2e, 0x76, 0x31, 0x1a, 0x2a, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2f, 0x73, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x2f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2f, 0x76, 0x33,
+	0x2f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e,
+	0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17,
+	0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74,
+	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd2, 0x04, 0x0a, 0x14, 0x4d, 0x6f, 0x6e, 0x69,
+	0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
+	0x12, 0x1b, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07,
+	0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x12, 0x21, 0x0a,
+	0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07,
+	0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x12, 0x4c, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x32, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61,
+	0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f,
+	0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x54,
+	0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x4f,
+	0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37,
+	0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c,
+	0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e,
+	0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65,
+	0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a,
+	0x9f, 0x02, 0x0a, 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20,
+	0x01, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01,
+	0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72,
+	0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02,
+	0x20, 0x01, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d,
+	0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x56,
+	0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e,
+	0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c,
+	0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e,
+	0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x54, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06,
+	0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
+	0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
+	0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xe6, 0x03, 0x0a,
+	0x24, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67,
+	0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x89, 0x01, 0x0a, 0x1a, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x4d,
+	0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d,
+	0x65, 0x6e, 0x74, 0x73, 0x12, 0x31, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76,
+	0x33, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72,
+	0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76,
+	0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30,
+	0x01, 0x12, 0x80, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x6f, 0x6e, 0x69,
+	0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
+	0x73, 0x12, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44,
+	0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x2d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e,
+	0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x69, 0x73,
+	0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x28, 0x01, 0x30, 0x01, 0x12, 0xae, 0x01, 0x0a, 0x1a, 0x46, 0x65, 0x74, 0x63, 0x68, 0x4d, 0x6f,
+	0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x12, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33,
+	0x2e, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x2d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44,
+	0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x33, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x24, 0x22, 0x22, 0x2f, 0x76, 0x33, 0x2f, 0x64, 0x69,
+	0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x3a, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69,
+	0x6e, 0x67, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x82, 0xd3, 0xe4, 0x93,
+	0x02, 0x03, 0x3a, 0x01, 0x2a, 0x42, 0x04, 0x5a, 0x02, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
+}
+
+var (
+	file_observability_v1_mads_proto_rawDescOnce sync.Once
+	file_observability_v1_mads_proto_rawDescData = file_observability_v1_mads_proto_rawDesc
+)
+
+func file_observability_v1_mads_proto_rawDescGZIP() []byte {
+	file_observability_v1_mads_proto_rawDescOnce.Do(func() {
+		file_observability_v1_mads_proto_rawDescData = protoimpl.X.CompressGZIP(file_observability_v1_mads_proto_rawDescData)
+	})
+	return file_observability_v1_mads_proto_rawDescData
+}
+
+var file_observability_v1_mads_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_observability_v1_mads_proto_goTypes = []interface{}{
+	(*MonitoringAssignment)(nil),        // 0: kuma.observability.v1.MonitoringAssignment
+	(*MonitoringAssignment_Target)(nil), // 1: kuma.observability.v1.MonitoringAssignment.Target
+	nil,                                 // 2: kuma.observability.v1.MonitoringAssignment.LabelsEntry
+	nil,                                 // 3: kuma.observability.v1.MonitoringAssignment.Target.LabelsEntry
+	(*v3.DeltaDiscoveryRequest)(nil),    // 4: envoy.service.discovery.v3.DeltaDiscoveryRequest
+	(*v3.DiscoveryRequest)(nil),         // 5: envoy.service.discovery.v3.DiscoveryRequest
+	(*v3.DeltaDiscoveryResponse)(nil),   // 6: envoy.service.discovery.v3.DeltaDiscoveryResponse
+	(*v3.DiscoveryResponse)(nil),        // 7: envoy.service.discovery.v3.DiscoveryResponse
+}
+var file_observability_v1_mads_proto_depIdxs = []int32{
+	1, // 0: kuma.observability.v1.MonitoringAssignment.targets:type_name -> kuma.observability.v1.MonitoringAssignment.Target
+	2, // 1: kuma.observability.v1.MonitoringAssignment.labels:type_name -> kuma.observability.v1.MonitoringAssignment.LabelsEntry
+	3, // 2: kuma.observability.v1.MonitoringAssignment.Target.labels:type_name -> kuma.observability.v1.MonitoringAssignment.Target.LabelsEntry
+	4, // 3: kuma.observability.v1.MonitoringAssignmentDiscoveryService.DeltaMonitoringAssignments:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest
+	5, // 4: kuma.observability.v1.MonitoringAssignmentDiscoveryService.StreamMonitoringAssignments:input_type -> envoy.service.discovery.v3.DiscoveryRequest
+	5, // 5: kuma.observability.v1.MonitoringAssignmentDiscoveryService.FetchMonitoringAssignments:input_type -> envoy.service.discovery.v3.DiscoveryRequest
+	6, // 6: kuma.observability.v1.MonitoringAssignmentDiscoveryService.DeltaMonitoringAssignments:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse
+	7, // 7: kuma.observability.v1.MonitoringAssignmentDiscoveryService.StreamMonitoringAssignments:output_type -> envoy.service.discovery.v3.DiscoveryResponse
+	7, // 8: kuma.observability.v1.MonitoringAssignmentDiscoveryService.FetchMonitoringAssignments:output_type -> envoy.service.discovery.v3.DiscoveryResponse
+	6, // [6:9] is the sub-list for method output_type
+	3, // [3:6] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_observability_v1_mads_proto_init() }
+func file_observability_v1_mads_proto_init() {
+	if File_observability_v1_mads_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_observability_v1_mads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MonitoringAssignment); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_observability_v1_mads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MonitoringAssignment_Target); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_observability_v1_mads_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_observability_v1_mads_proto_goTypes,
+		DependencyIndexes: file_observability_v1_mads_proto_depIdxs,
+		MessageInfos:      file_observability_v1_mads_proto_msgTypes,
+	}.Build()
+	File_observability_v1_mads_proto = out.File
+	file_observability_v1_mads_proto_rawDesc = nil
+	file_observability_v1_mads_proto_goTypes = nil
+	file_observability_v1_mads_proto_depIdxs = nil
+}
+
+// MonitoringAssignmentDiscoveryServiceServer is the server API for MonitoringAssignmentDiscoveryService service.
+type MonitoringAssignmentDiscoveryServiceServer interface {
+	// HTTP
+	FetchMonitoringAssignments(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error)
+}

discovery/xds/kuma_test.go

@@ -0,0 +1,340 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+var (
+	kumaConf KumaSDConfig = sdConf
+
+	testKumaMadsV1Resources = []*MonitoringAssignment{
+		{
+			Mesh:    "metrics",
+			Service: "prometheus",
+			Targets: []*MonitoringAssignment_Target{
+				{
+					Name:        "prometheus-01",
+					Scheme:      "http",
+					Address:     "10.1.4.32:9090",
+					MetricsPath: "/custom-metrics",
+					Labels: map[string]string{
+						"commit_hash": "620506a88",
+					},
+				},
+				{
+					Name:    "prometheus-02",
+					Scheme:  "http",
+					Address: "10.1.4.33:9090",
+					Labels: map[string]string{
+						"commit_hash": "3513bba00",
+					},
+				},
+			},
+			Labels: map[string]string{
+				"kuma.io/zone": "us-east-1",
+				"team":         "infra",
+			},
+		},
+		{
+			Mesh:    "metrics",
+			Service: "grafana",
+			Targets: []*MonitoringAssignment_Target{},
+			Labels: map[string]string{
+				"kuma.io/zone": "us-east-1",
+				"team":         "infra",
+			},
+		},
+		{
+			Mesh:    "data",
+			Service: "elasticsearch",
+			Targets: []*MonitoringAssignment_Target{
+				{
+					Name:    "elasticsearch-01",
+					Scheme:  "http",
+					Address: "10.1.1.1",
+					Labels: map[string]string{
+						"role": "ml",
+					},
+				},
+			},
+		},
+	}
+)
+
+func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.DiscoveryResponse, error) {
+	serialized := make([]*anypb.Any, len(resources))
+	for i, res := range resources {
+		data, err := proto.Marshal(res)
+
+		if err != nil {
+			return nil, err
+		}
+
+		serialized[i] = &anypb.Any{
+			TypeUrl: KumaMadsV1ResourceTypeURL,
+			Value:   data,
+		}
+	}
+	return &v3.DiscoveryResponse{
+		TypeUrl:   KumaMadsV1ResourceTypeURL,
+		Resources: serialized,
+	}, nil
+}
+
+func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) {
+	kd, err := NewKumaHTTPDiscovery(&c, nopLogger)
+	if err != nil {
+		return nil, err
+	}
+
+	pd, ok := kd.(*fetchDiscovery)
+	if !ok {
+		return nil, errors.New("not a fetchDiscovery")
+	}
+	return pd, nil
+}
+
+func TestKumaMadsV1ResourceParserInvalidTypeURL(t *testing.T) {
+	resources := make([]*anypb.Any, 0)
+	groups, err := kumaMadsV1ResourceParser(resources, "type.googleapis.com/some.api.v1.Monitoring")
+	require.Nil(t, groups)
+	require.Error(t, err)
+}
+
+func TestKumaMadsV1ResourceParserEmptySlice(t *testing.T) {
+	resources := make([]*anypb.Any, 0)
+	groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL)
+	require.Len(t, groups, 0)
+	require.NoError(t, err)
+}
+
+func TestKumaMadsV1ResourceParserValidResources(t *testing.T) {
+	res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...)
+	require.NoError(t, err)
+
+	groups, err := kumaMadsV1ResourceParser(res.Resources, KumaMadsV1ResourceTypeURL)
+	require.NoError(t, err)
+	require.Len(t, groups, 3)
+
+	expectedGroup1 := &targetgroup.Group{
+		Targets: []model.LabelSet{
+			{
+				"__address__":                   "10.1.4.32:9090",
+				"__meta_kuma_label_commit_hash": "620506a88",
+				"__meta_kuma_dataplane":         "prometheus-01",
+				"__metrics_path__":              "/custom-metrics",
+				"__scheme__":                    "http",
+				"instance":                      "prometheus-01",
+			},
+			{
+				"__address__":                   "10.1.4.33:9090",
+				"__meta_kuma_label_commit_hash": "3513bba00",
+				"__meta_kuma_dataplane":         "prometheus-02",
+				"__metrics_path__":              "",
+				"__scheme__":                    "http",
+				"instance":                      "prometheus-02",
+			},
+		},
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":               "metrics",
+			"__meta_kuma_service":            "prometheus",
+			"__meta_kuma_label_team":         "infra",
+			"__meta_kuma_label_kuma_io_zone": "us-east-1",
+		},
+	}
+	require.Equal(t, expectedGroup1, groups[0])
+
+	expectedGroup2 := &targetgroup.Group{
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":               "metrics",
+			"__meta_kuma_service":            "grafana",
+			"__meta_kuma_label_team":         "infra",
+			"__meta_kuma_label_kuma_io_zone": "us-east-1",
+		},
+	}
+	require.Equal(t, expectedGroup2, groups[1])
+
+	expectedGroup3 := &targetgroup.Group{
+		Targets: []model.LabelSet{
+			{
+				"__address__":            "10.1.1.1",
+				"__meta_kuma_label_role": "ml",
+				"__meta_kuma_dataplane":  "elasticsearch-01",
+				"__metrics_path__":       "",
+				"__scheme__":             "http",
+				"instance":               "elasticsearch-01",
+			},
+		},
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":    "data",
+			"__meta_kuma_service": "elasticsearch",
+		},
+	}
+	require.Equal(t, expectedGroup3, groups[2])
+}
+
+func TestKumaMadsV1ResourceParserInvalidResources(t *testing.T) {
+	data, err := protoJSONMarshalOptions.Marshal(&MonitoringAssignment_Target{})
+	require.NoError(t, err)
+
+	resources := []*anypb.Any{{
+		TypeUrl: KumaMadsV1ResourceTypeURL,
+		Value:   data,
+	}}
+	groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL)
+	require.Nil(t, groups)
+	require.Error(t, err)
+
+	require.Contains(t, err.Error(), "cannot parse")
+}
+
+func TestNewKumaHTTPDiscovery(t *testing.T) {
+	kd, err := newKumaTestHTTPDiscovery(kumaConf)
+	require.NoError(t, err)
+	require.NotNil(t, kd)
+
+	resClient, ok := kd.client.(*HTTPResourceClient)
+	require.True(t, ok)
+	require.Equal(t, kumaConf.Server, resClient.Server())
+	require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL())
+	require.NotEmpty(t, resClient.ID())
+	require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType)
+}
+
+func TestKumaHTTPDiscoveryRefresh(t *testing.T) {
+	s := createTestHTTPServer(t, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
+		if request.VersionInfo == "1" {
+			return nil, nil
+		}
+
+		res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...)
+		require.NoError(t, err)
+
+		res.VersionInfo = "1"
+		res.Nonce = "abc"
+
+		return res, nil
+	})
+	defer s.Close()
+
+	cfgString := fmt.Sprintf(`
+---
+server: %s
+refresh_interval: 10s
+tls_config:
+  insecure_skip_verify: true
+`, s.URL)
+
+	var cfg KumaSDConfig
+	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
+
+	kd, err := newKumaTestHTTPDiscovery(cfg)
+	require.NoError(t, err)
+	require.NotNil(t, kd)
+
+	ch := make(chan []*targetgroup.Group, 1)
+	kd.poll(context.Background(), ch)
+
+	groups := <-ch
+	require.Len(t, groups, 3)
+
+	expectedGroup1 := &targetgroup.Group{
+		Source: "kuma",
+		Targets: []model.LabelSet{
+			{
+				"__address__":                   "10.1.4.32:9090",
+				"__meta_kuma_label_commit_hash": "620506a88",
+				"__meta_kuma_dataplane":         "prometheus-01",
+				"__metrics_path__":              "/custom-metrics",
+				"__scheme__":                    "http",
+				"instance":                      "prometheus-01",
+			},
+			{
+				"__address__":                   "10.1.4.33:9090",
+				"__meta_kuma_label_commit_hash": "3513bba00",
+				"__meta_kuma_dataplane":         "prometheus-02",
+				"__metrics_path__":              "",
+				"__scheme__":                    "http",
+				"instance":                      "prometheus-02",
+			},
+		},
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":               "metrics",
+			"__meta_kuma_service":            "prometheus",
+			"__meta_kuma_label_team":         "infra",
+			"__meta_kuma_label_kuma_io_zone": "us-east-1",
+		},
+	}
+	require.Equal(t, expectedGroup1, groups[0])
+
+	expectedGroup2 := &targetgroup.Group{
+		Source: "kuma",
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":               "metrics",
+			"__meta_kuma_service":            "grafana",
+			"__meta_kuma_label_team":         "infra",
+			"__meta_kuma_label_kuma_io_zone": "us-east-1",
+		},
+	}
+	require.Equal(t, expectedGroup2, groups[1])
+
+	expectedGroup3 := &targetgroup.Group{
+		Source: "kuma",
+		Targets: []model.LabelSet{
+			{
+				"__address__":            "10.1.1.1",
+				"__meta_kuma_label_role": "ml",
+				"__meta_kuma_dataplane":  "elasticsearch-01",
+				"__metrics_path__":       "",
+				"__scheme__":             "http",
+				"instance":               "elasticsearch-01",
+			},
+		},
+		Labels: model.LabelSet{
+			"__meta_kuma_mesh":    "data",
+			"__meta_kuma_service": "elasticsearch",
+		},
+	}
+	require.Equal(t, expectedGroup3, groups[2])
+
+	// Should skip the next update.
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(1 * time.Second)
+		cancel()
+	}()
+
+	kd.poll(ctx, ch)
+	select {
+	case <-ctx.Done():
+		return
+	case <-ch:
+		require.Fail(t, "no update expected")
+	}
+}

discovery/xds/xds.go

@@ -0,0 +1,176 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"context"
+	"github.com/prometheus/common/model"
+	"time"
+
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/config"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+const (
+	// Constants for instrumentation.
+	namespace = "prometheus"
+)
+
+// ProtocolVersion is the xDS protocol version.
+type ProtocolVersion string
+
+const (
+	ProtocolV3 = ProtocolVersion("v3")
+)
+
+type HTTPConfig struct {
+	config.HTTPClientConfig `yaml:",inline"`
+}
+
+// SDConfig is a base config for xDS-based SD mechanisms.
+type SDConfig struct {
+	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
+	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
+	FetchTimeout     model.Duration          `yaml:"fetch_timeout,omitempty"`
+	Server           string                  `yaml:"server,omitempty"`
+}
+
+// mustRegisterMessage registers the provided message type in the typeRegistry, and panics
+// if there is an error.
+func mustRegisterMessage(typeRegistry *protoregistry.Types, mt protoreflect.MessageType) {
+	if err := typeRegistry.RegisterMessage(mt); err != nil {
+		panic(err)
+	}
+}
+
+func init() {
+	// Register top-level SD Configs.
+	discovery.RegisterConfig(&KumaSDConfig{})
+
+	// Register metrics.
+	prometheus.MustRegister(kumaFetchDuration, kumaFetchSkipUpdateCount, kumaFetchFailuresCount)
+
+	// Register protobuf types that need to be marshalled/ unmarshalled.
+	mustRegisterMessage(protoTypes, (&v3.DiscoveryRequest{}).ProtoReflect().Type())
+	mustRegisterMessage(protoTypes, (&v3.DiscoveryResponse{}).ProtoReflect().Type())
+	mustRegisterMessage(protoTypes, (&MonitoringAssignment{}).ProtoReflect().Type())
+}
+
+var (
+	protoTypes            = new(protoregistry.Types)
+	protoUnmarshalOptions = proto.UnmarshalOptions{
+		DiscardUnknown: true,       // Only want known fields.
+		Merge:          true,       // Always using new messages.
+		Resolver:       protoTypes, // Only want known types.
+	}
+	protoJSONUnmarshalOptions = protojson.UnmarshalOptions{
+		DiscardUnknown: true,       // Only want known fields.
+		Resolver:       protoTypes, // Only want known types.
+	}
+	protoJSONMarshalOptions = protojson.MarshalOptions{
+		UseProtoNames: true,
+		Resolver:      protoTypes, // Only want known types.
+	}
+)
+
+type resourceParser func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error)
+
+// fetchDiscovery implements long-polling via xDS Fetch REST-JSON.
+type fetchDiscovery struct {
+	client ResourceClient
+	source string
+
+	refreshInterval time.Duration
+
+	parseResources resourceParser
+	logger         log.Logger
+
+	fetchDuration        prometheus.Observer
+	fetchSkipUpdateCount prometheus.Counter
+	fetchFailuresCount   prometheus.Counter
+}
+
+func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	defer d.client.Close()
+
+	ticker := time.NewTicker(d.refreshInterval)
+
+	for {
+		select {
+		case <-ctx.Done():
+			ticker.Stop()
+			return
+		default:
+			d.poll(ctx, ch)
+			<-ticker.C
+		}
+	}
+}
+
+func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	t0 := time.Now()
+	response, err := d.client.Fetch(ctx)
+	elapsed := time.Since(t0)
+	d.fetchDuration.Observe(elapsed.Seconds())
+
+	// Check the context before in order to exit early.
+	select {
+	case <-ctx.Done():
+		return
+	default:
+	}
+
+	if err != nil {
+		level.Error(d.logger).Log("msg", "error parsing resources", "err", err)
+		d.fetchFailuresCount.Inc()
+		return
+	}
+
+	if response == nil {
+		// No update needed.
+		d.fetchSkipUpdateCount.Inc()
+		return
+	}
+
+	parsedGroups, err := d.parseResources(response.Resources, response.TypeUrl)
+	if err != nil {
+		level.Error(d.logger).Log("msg", "error parsing resources", "err", err)
+		d.fetchFailuresCount.Inc()
+		return
+	}
+
+	for _, group := range parsedGroups {
+		group.Source = d.source
+	}
+
+	level.Debug(d.logger).Log("msg", "updated to version", "version", response.VersionInfo, "groups", len(parsedGroups))
+
+	// Check the context before sending an update on the channel.
+	select {
+	case <-ctx.Done():
+		return
+	case ch <- parsedGroups:
+	}
+}

discovery/xds/xds_test.go

@@ -0,0 +1,201 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+var (
+	sdConf = SDConfig{
+		Server:          "http://127.0.0.1",
+		RefreshInterval: model.Duration(10 * time.Second),
+	}
+
+	testFetchFailuresCount = prometheus.NewCounter(
+		prometheus.CounterOpts{})
+	testFetchSkipUpdateCount = prometheus.NewCounter(
+		prometheus.CounterOpts{})
+	testFetchDuration = prometheus.NewSummary(
+		prometheus.SummaryOpts{},
+	)
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+type discoveryResponder func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error)
+
+func createTestHTTPServer(t *testing.T, responder discoveryResponder) *httptest.Server {
+	return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Validate req MIME types.
+		require.Equal(t, "application/json", r.Header.Get("Content-Type"))
+		require.Equal(t, "application/json", r.Header.Get("Accept"))
+
+		body, err := ioutil.ReadAll(r.Body)
+		defer func() {
+			_, _ = io.Copy(ioutil.Discard, r.Body)
+			_ = r.Body.Close()
+		}()
+		require.NotEmpty(t, body)
+		require.NoError(t, err)
+
+		// Validate discovery request.
+		discoveryReq := &v3.DiscoveryRequest{}
+		err = protoJSONUnmarshalOptions.Unmarshal(body, discoveryReq)
+		require.NoError(t, err)
+
+		discoveryRes, err := responder(discoveryReq)
+		if err != nil {
+			w.WriteHeader(500)
+			return
+		}
+
+		if discoveryRes == nil {
+			w.WriteHeader(304)
+			return
+		}
+
+		w.WriteHeader(200)
+		data, err := protoJSONMarshalOptions.Marshal(discoveryRes)
+		require.NoError(t, err)
+
+		_, err = w.Write(data)
+		require.NoError(t, err)
+	}))
+}
+
+func constantResourceParser(groups []*targetgroup.Group, err error) resourceParser {
+	return func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error) {
+		return groups, err
+	}
+}
+
+var nopLogger = log.NewNopLogger()
+
+type testResourceClient struct {
+	resourceTypeURL string
+	server          string
+	protocolVersion ProtocolVersion
+	fetch           func(ctx context.Context) (*v3.DiscoveryResponse, error)
+}
+
+func (rc testResourceClient) ResourceTypeURL() string {
+	return rc.resourceTypeURL
+}
+
+func (rc testResourceClient) Server() string {
+	return rc.server
+}
+
+func (rc testResourceClient) Fetch(ctx context.Context) (*v3.DiscoveryResponse, error) {
+	return rc.fetch(ctx)
+}
+
+func (rc testResourceClient) ID() string {
+	return "test-client"
+}
+
+func (rc testResourceClient) Close() {
+}
+
+func TestPollingRefreshSkipUpdate(t *testing.T) {
+	rc := &testResourceClient{
+		fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) {
+			return nil, nil
+		},
+	}
+	pd := &fetchDiscovery{
+		client:               rc,
+		logger:               nopLogger,
+		fetchDuration:        testFetchDuration,
+		fetchFailuresCount:   testFetchFailuresCount,
+		fetchSkipUpdateCount: testFetchSkipUpdateCount,
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(1 * time.Second)
+		cancel()
+	}()
+
+	ch := make(chan []*targetgroup.Group, 1)
+	pd.poll(ctx, ch)
+	select {
+	case <-ctx.Done():
+		return
+	case <-ch:
+		require.Fail(t, "no update expected")
+	}
+}
+
+func TestPollingRefreshAttachesGroupMetadata(t *testing.T) {
+	server := "http://198.161.2.0"
+	source := "test"
+	rc := &testResourceClient{
+		server:          server,
+		protocolVersion: ProtocolV3,
+		fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) {
+			return &v3.DiscoveryResponse{}, nil
+		},
+	}
+	pd := &fetchDiscovery{
+		source:               source,
+		client:               rc,
+		logger:               nopLogger,
+		fetchDuration:        testFetchDuration,
+		fetchFailuresCount:   testFetchFailuresCount,
+		fetchSkipUpdateCount: testFetchSkipUpdateCount,
+		parseResources: constantResourceParser([]*targetgroup.Group{
+			{},
+			{
+				Source: "a-custom-source",
+				Labels: model.LabelSet{
+					"__meta_custom_xds_label": "a-value",
+				},
+			},
+		}, nil),
+	}
+	ch := make(chan []*targetgroup.Group, 1)
+	pd.poll(context.Background(), ch)
+	groups := <-ch
+	require.NotNil(t, groups)
+
+	require.Len(t, groups, 2)
+
+	for _, group := range groups {
+		require.Equal(t, source, group.Source)
+	}
+
+	group2 := groups[1]
+	require.Contains(t, group2.Labels, model.LabelName("__meta_custom_xds_label"))
+	require.Equal(t, model.LabelValue("a-value"), group2.Labels["__meta_custom_xds_label"])
+}

docs/configuration/configuration.md

@@ -248,6 +248,10 @@ http_sd_configs:
 kubernetes_sd_configs:
   [ - <kubernetes_sd_config> ... ]
 
+# List of Kuma service discovery configurations.
+kuma_sd_configs:
+  [ - <kuma_sd_config> ... ]
+
 # List of Lightsail service discovery configurations.
 lightsail_sd_configs:
   [ - <lightsail_sd_config> ... ]
@@ -382,6 +386,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
 * `__meta_azure_machine_id`: the machine ID
 * `__meta_azure_machine_location`: the location the machine runs in
 * `__meta_azure_machine_name`: the machine name
+* `__meta_azure_machine_computer_name`: the machine computer name
 * `__meta_azure_machine_os_type`: the machine operating system
 * `__meta_azure_machine_private_ip`: the machine's private IP
 * `__meta_azure_machine_public_ip`: the machine's public IP if it exists
@@ -635,6 +640,9 @@ tls_config:
 # tasks and services that don't have published ports.
 [ port: <int> | default = 80 ]
 
+# The host to use if the container is in host networking mode.
+[ host_networking_host: <string> | default = "localhost" ]
+
 # Optional filters to limit the discovery process to a subset of available
 # resources.
 # The available filters are listed in the upstream documentation:
@@ -893,6 +901,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
 * `__meta_ec2_ami`: the EC2 Amazon Machine Image
 * `__meta_ec2_architecture`: the architecture of the instance
 * `__meta_ec2_availability_zone`: the availability zone in which the instance is running
+* `__meta_ec2_availability_zone_id`: the [availability zone ID](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) in which the instance is running
 * `__meta_ec2_instance_id`: the EC2 instance ID
 * `__meta_ec2_instance_lifecycle`: the lifecycle of the EC2 instance, set only for 'spot' or 'scheduled' instances, absent otherwise
 * `__meta_ec2_instance_state`: the state of the EC2 instance
@@ -1131,6 +1140,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
 * `__meta_gce_metadata_<name>`: each metadata item of the instance
 * `__meta_gce_network`: the network URL of the instance
 * `__meta_gce_private_ip`: the private IP address of the instance
+* `__meta_gce_interface_ipv4_<name>`: IPv4 address of each named interface
 * `__meta_gce_project`: the GCP project in which the instance is running
 * `__meta_gce_public_ip`: the public IP address of the instance, if present
 * `__meta_gce_subnetwork`: the subnetwork URL of the instance
@@ -1211,6 +1221,7 @@ The labels below are only available for targets with `role` set to `hcloud`:
 * `__meta_hetzner_hcloud_disk_size_gb`: the disk size of the server (in GB)
 * `__meta_hetzner_hcloud_private_ipv4_<networkname>`: the private ipv4 address of the server within a given network
 * `__meta_hetzner_hcloud_label_<labelname>`: each label of the server
+* `__meta_hetzner_hcloud_labelpresent_<labelname>`: `true` for each label of the server
 
 The labels below are only available for targets with `role` set to `robot`:
 
@@ -1545,6 +1556,74 @@ for a detailed example of configuring Prometheus for Kubernetes.
 You may wish to check out the 3rd party [Prometheus Operator](https://github.com/coreos/prometheus-operator),
 which automates the Prometheus setup on top of Kubernetes.
 
+### `<kuma_sd_config>`
+
+Kuma SD configurations allow retrieving scrape target from the [Kuma](https://kuma.io) control plane.
+
+This SD discovers "monitoring assignments" based on Kuma [Dataplane Proxies](https://kuma.io/docs/latest/documentation/dps-and-data-model),
+via the MADS v1 (Monitoring Assignment Discovery Service) xDS API, and will create a target for each proxy
+inside a Prometheus-enabled mesh.
+
+The following meta labels are available for each target:
+
+* `__meta_kuma_mesh`: the name of the proxy's Mesh 
+* `__meta_kuma_dataplane`: the name of the proxy
+* `__meta_kuma_service`: the name of the proxy's associated Service
+* `__meta_kuma_label_<tagname>`: each tag of the proxy
+
+See below for the configuration options for Kuma MonitoringAssignment discovery:
+
+```yaml
+# Address of the Kuma Control Plane's MADS xDS server.
+server: <string>
+
+# The time to wait between polling update requests.
+[ refresh_interval: <duration> | default = 30s ]
+
+# The time after which the monitoring assignments are refreshed.
+[ fetch_timeout: <duration> | default = 2m ]
+
+# Optional proxy URL.
+[ proxy_url: <string> ]
+
+# TLS configuration.
+tls_config:
+  [ <tls_config> ]
+
+# Authentication information used to authenticate to the Docker daemon.
+# Note that `basic_auth` and `authorization` options are
+# mutually exclusive.
+# password and password_file are mutually exclusive.
+
+# Optional HTTP basic authentication information.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# Optional the `Authorization` header configuration.
+authorization:
+  # Sets the authentication type.
+  [ type: <string> | default: Bearer ]
+  # Sets the credentials. It is mutually exclusive with
+  # `credentials_file`.
+  [ credentials: <secret> ]
+  # Sets the credentials with the credentials read from the configured file.
+  # It is mutually exclusive with `credentials`.
+  [ credentials_file: <filename> ]
+
+# Optional OAuth 2.0 configuration.
+# Cannot be used at the same time as basic_auth or authorization.
+oauth2:
+  [ <oauth2> ]
+
+# Configure whether HTTP requests follow HTTP 3xx redirects.
+[ follow_redirects: <bool> | default = true ]
+```
+
+The [relabeling phase](#relabel_config) is the preferred and more powerful way
+to filter proxies and user-defined tags.
+
 ### `<lightsail_sd_config>`
 
 Lightsail SD configurations allow retrieving scrape targets from [AWS Lightsail](https://aws.amazon.com/lightsail/)
@@ -1627,7 +1706,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
 # Note that `basic_auth` and `authorization` options are
 # mutually exclusive.
 # password and password_file are mutually exclusive.
-# Note: Linode APIv4 Token must be created with scopes: 'linodes:read_only' and 'ips:read_only'
+# Note: Linode APIv4 Token must be created with scopes: 'linodes:read_only', 'ips:read_only', and 'events:read_only'
 
 # Optional HTTP basic authentication information, not currently supported by Linode APIv4.
 basic_auth:

docs/configuration/https.md

@@ -36,6 +36,9 @@ tls_server_config:
   # Server policy for client authentication. Maps to ClientAuth Policies.
   # For more detail on clientAuth options:
   # https://golang.org/pkg/crypto/tls/#ClientAuthType
+  #
+  # NOTE: If you want to enable client authentication, you need to use
+  # RequireAndVerifyClientCert. Other values are insecure.
   [ client_auth_type: <string> | default = "NoClientCert" ]
 
   # CA certificate for client certificate authentication to the server.

docs/configuration/unit_testing_rules.md

@@ -78,9 +78,13 @@ series: <string>
 # Expanding notation:
 #     'a+bxc' becomes 'a a+b a+(2*b) a+(3*b) … a+(c*b)'
 #     'a-bxc' becomes 'a a-b a-(2*b) a-(3*b) … a-(c*b)'
+# There are special values to indicate missing and stale samples:
+#    '_' represents a missing sample from scrape
+#    'stale' indicates a stale sample
 # Examples:
 #     1. '-2+4x3' becomes '-2 2 6 10'
 #     2. ' 1-2x4' becomes '1 -1 -3 -5 -7'
+#     3. ' 1 _x3 stale' becomes '1 _ _ _ stale'
 values: <string>
 ```
 

docs/feature_flags.md

@@ -1,9 +1,9 @@
 ---
-title: Disabled Features
+title: Feature Flags
 sort_rank: 11
 ---
 
-# Disabled Features
+# Feature Flags
 
 Here is a list of features that are disabled by default since they are breaking changes or are considered experimental.
 Their behaviour can change in future releases which will be communicated via the [release changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md).

docs/querying/api.md

@@ -145,8 +145,8 @@ POST /api/v1/query_range
 URL query parameters:
 
 - `query=<string>`: Prometheus expression query string.
-- `start=<rfc3339 | unix_timestamp>`: Start timestamp.
+- `start=<rfc3339 | unix_timestamp>`: Start timestamp, inclusive.
-- `end=<rfc3339 | unix_timestamp>`: End timestamp.
+- `end=<rfc3339 | unix_timestamp>`: End timestamp, inclusive.
 - `step=<duration | float>`: Query resolution step width in `duration` format or float number of seconds.
 - `timeout=<duration>`: Evaluation timeout. Optional. Defaults to and
    is capped by the value of the `-query.timeout` flag.

docs/querying/functions.md

@@ -430,6 +430,7 @@ over time and return an instant vector with per-series aggregation results:
 * `stddev_over_time(range-vector)`: the population standard deviation of the values in the specified interval.
 * `stdvar_over_time(range-vector)`: the population standard variance of the values in the specified interval.
 * `last_over_time(range-vector)`: the most recent point value in specified interval.
+* `present_over_time(range-vector)`: the value 1 for any series in the specified interval.
 
 Note that all values in the specified interval have the same weight in the
 aggregation even if the values are not equally spaced throughout the interval.

docs/storage.md

@@ -82,7 +82,7 @@ Prometheus has several flags that configure local storage. The most important ar
 
 * `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`.
 * `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
-* `--storage.tsdb.retention.size`: [EXPERIMENTAL] The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. This flag is experimental and may change in future releases. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB"
+* `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB"
 * `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`.
 * `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL.
 
@@ -157,6 +157,16 @@ promtool tsdb create-blocks-from openmetrics <input file> [<output directory>]
 
 After the creation of the blocks, move it to the data directory of Prometheus. If there is an overlap with the existing blocks in Prometheus, the flag `--storage.tsdb.allow-overlapping-blocks` needs to be set. Note that any backfilled data is subject to the retention configured for your Prometheus server (by time or size).
 
+#### Longer Block Durations
+
+By default, the promtool will use the default block duration (2h) for the blocks; this behavior is the most generally applicable and correct. However, when backfilling data over a long range of times, it may be advantageous to use a larger value for the block duration to backfill faster and prevent additional compactions by TSDB later.
+
+The `--max-block-duration` flag allows the user to configure a maximum duration of blocks. The backfilling tool will pick a suitable block duration no larger than this.
+
+While larger blocks may improve the performance of backfilling large datasets, drawbacks exist as well. Time-based retention policies must keep the entire block around if even one sample of the (potentially large) block is still within the retention policy. Conversely, size-based retention policies will remove the entire block even if the TSDB only goes over the size limit in a minor way.
+
+Therefore, backfilling with few blocks, thereby choosing a larger block duration, must be done with care and is not recommended for any production instances.
+
 ## Backfilling for Recording Rules
 
 ### Overview

documentation/examples/custom-sd/adapter-usage/main.go

@@ -100,13 +100,17 @@ func (d *discovery) parseServiceNodes(resp *http.Response, name string) (*target
 		Labels: make(model.LabelSet),
 	}
 
-	dec := json.NewDecoder(resp.Body)
 	defer func() {
 		io.Copy(ioutil.Discard, resp.Body)
 		resp.Body.Close()
 	}()
-	err := dec.Decode(&nodes)
 
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(b, &nodes)
 	if err != nil {
 		return &tgroup, err
 	}
@@ -165,8 +169,8 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			continue
 		}
 
-		dec := json.NewDecoder(resp.Body)
+		b, err := ioutil.ReadAll(resp.Body)
-		err = dec.Decode(&srvs)
+		io.Copy(ioutil.Discard, resp.Body)
 		resp.Body.Close()
 		if err != nil {
 			level.Error(d.logger).Log("msg", "Error reading services list", "err", err)
@@ -174,6 +178,14 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			continue
 		}
 
+		err = json.Unmarshal(b, &srvs)
+		resp.Body.Close()
+		if err != nil {
+			level.Error(d.logger).Log("msg", "Error parsing services list", "err", err)
+			time.Sleep(time.Duration(d.refreshInterval) * time.Second)
+			continue
+		}
+
 		var tgs []*targetgroup.Group
 		// Note that we treat errors when querying specific consul services as fatal for this
 		// iteration of the time.Tick loop. It's better to have some stale targets than an incomplete
@@ -191,6 +203,7 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				level.Error(d.logger).Log("msg", "Error getting services nodes", "service", name, "err", err)
 				break
 			}
+
 			tg, err := d.parseServiceNodes(resp, name)
 			if err != nil {
 				level.Error(d.logger).Log("msg", "Error parsing services nodes", "service", name, "err", err)

documentation/examples/prometheus-linode.yml

@@ -11,7 +11,7 @@ scrape_configs:
   - job_name: "node"
     linode_sd_configs:
       - authorization:
-        credentials: "<replace with a Personal Access Token with linodes:read_only + ips:read_only access>"
+        credentials: "<replace with a Personal Access Token with linodes:read_only, ips:read_only, and events:read_only access>"
     relabel_configs:
       # Only scrape targets that have a tag 'monitoring'.
       - source_labels: [__meta_linode_tags]

go.mod

@@ -3,83 +3,82 @@ module github.com/prometheus/prometheus
 go 1.14
 
 require (
-	github.com/Azure/azure-sdk-for-go v55.2.0+incompatible
+	github.com/Azure/azure-sdk-for-go v55.8.0+incompatible
 	github.com/Azure/go-autorest/autorest v0.11.19
 	github.com/Azure/go-autorest/autorest/adal v0.9.14
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
-	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
-	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
-	github.com/aws/aws-sdk-go v1.38.60
+	github.com/aws/aws-sdk-go v1.40.10
 	github.com/cespare/xxhash/v2 v2.1.1
-	github.com/containerd/containerd v1.4.3 // indirect
+	github.com/containerd/containerd v1.5.4 // indirect
+	github.com/dennwc/varint v1.0.0
 	github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245
-	github.com/digitalocean/godo v1.62.0
+	github.com/digitalocean/godo v1.64.2
-	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0
+	github.com/envoyproxy/go-control-plane v0.9.9
+	github.com/envoyproxy/protoc-gen-validate v0.6.1
 	github.com/go-kit/log v0.1.0
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/go-openapi/strfmt v0.20.1
 	github.com/go-zookeeper/zk v1.0.2
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/snappy v0.0.3
+	github.com/golang/snappy v0.0.4
-	github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9
+	github.com/google/pprof v0.0.0-20210726183535-c50bf4fe5303
-	github.com/gophercloud/gophercloud v0.18.0
+	github.com/gophercloud/gophercloud v0.19.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/hashicorp/consul/api v1.8.1
+	github.com/hashicorp/consul/api v1.9.1
-	github.com/hetznercloud/hcloud-go v1.26.2
+	github.com/hetznercloud/hcloud-go v1.28.0
-	github.com/influxdata/influxdb v1.9.2
+	github.com/influxdata/influxdb v1.9.3
 	github.com/json-iterator/go v1.1.11
-	github.com/linode/linodego v0.28.5
+	github.com/linode/linodego v0.31.0
-	github.com/miekg/dns v1.1.42
+	github.com/miekg/dns v1.1.43
 	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
 	github.com/oklog/run v1.1.0
 	github.com/oklog/ulid v1.3.1
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opentracing-contrib/go-stdlib v1.0.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.22.2
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.29.0
+	github.com/prometheus/common v0.30.0
-	github.com/prometheus/exporter-toolkit v0.5.1
+	github.com/prometheus/exporter-toolkit v0.6.1
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
 	github.com/stretchr/testify v1.7.0
 	github.com/uber/jaeger-client-go v2.29.1+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
-	go.uber.org/atomic v1.8.0
+	go.uber.org/atomic v1.9.0
 	go.uber.org/goleak v1.1.10
-	golang.org/x/net v0.0.0-20210610132358-84b48f89b13b
+	golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985
-	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
+	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
-	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
-	golang.org/x/tools v0.1.3
+	golang.org/x/tools v0.1.5
-	google.golang.org/api v0.48.0
+	google.golang.org/api v0.51.0
+	google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify/fsnotify.v1 v1.4.7
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	gotest.tools/v3 v3.0.3 // indirect
+	k8s.io/api v0.21.3
-	k8s.io/api v0.21.1
+	k8s.io/apimachinery v0.21.3
-	k8s.io/apimachinery v0.21.1
+	k8s.io/client-go v0.21.3
-	k8s.io/client-go v0.21.1
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.9.0
+	k8s.io/klog/v2 v2.10.0
 )
 
 replace (
 	k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
-	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v2 v2.1.0
+	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.0.0
 )
 
 // Exclude linodego v1.0.0 as it is no longer published on github.

notifier/notifier_test.go

@@ -125,8 +125,16 @@ func TestHandlerSendAll(t *testing.T) {
 				w.WriteHeader(http.StatusInternalServerError)
 				return
 			}
+
+			b, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				err = errors.Errorf("error reading body: %v", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
 			var alerts []*Alert
-			err = json.NewDecoder(r.Body).Decode(&alerts)
+			err = json.Unmarshal(b, &alerts)
 			if err == nil {
 				err = alertsEqual(expected, alerts)
 			}
@@ -322,7 +330,13 @@ func TestHandlerQueuing(t *testing.T) {
 		select {
 		case expected := <-expectedc:
 			var alerts []*Alert
-			err := json.NewDecoder(r.Body).Decode(&alerts)
+
+			b, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				panic(err)
+			}
+
+			err = json.Unmarshal(b, &alerts)
 			if err == nil {
 				err = alertsEqual(expected, alerts)
 			}

pkg/textparse/openmetricsparse.go

@@ -343,6 +343,9 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
 				return EntryInvalid, err
 			}
+			if math.IsNaN(ts) || math.IsInf(ts, 0) {
+				return EntryInvalid, errors.New("invalid timestamp")
+			}
 			p.ts = int64(ts * 1000)
 			switch t3 := p.nextToken(); t3 {
 			case tLinebreak:
@@ -399,6 +402,9 @@ func (p *OpenMetricsParser) parseComment() error {
 		if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
 			return err
 		}
+		if math.IsNaN(ts) || math.IsInf(ts, 0) {
+			return errors.New("invalid exemplar timestamp")
+		}
 		p.exemplarTs = int64(ts * 1000)
 		switch t3 := p.nextToken(); t3 {
 		case tLinebreak:

pkg/textparse/openmetricsparse_test.go

@@ -502,6 +502,30 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 			input: `{b="c",} 1`,
 			err:   `"INVALID" "{" is not a valid start token`,
 		},
+		{
+			input: `a 1 NaN`,
+			err:   `invalid timestamp`,
+		},
+		{
+			input: `a 1 -Inf`,
+			err:   `invalid timestamp`,
+		},
+		{
+			input: `a 1 Inf`,
+			err:   `invalid timestamp`,
+		},
+		{
+			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 NaN",
+			err:   `invalid exemplar timestamp`,
+		},
+		{
+			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf",
+			err:   `invalid exemplar timestamp`,
+		},
+		{
+			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf",
+			err:   `invalid exemplar timestamp`,
+		},
 	}
 
 	for i, c := range cases {

promql/engine.go

@@ -117,6 +117,8 @@ type Query interface {
 	Stats() *stats.QueryTimers
 	// Cancel signals that a running query execution should be aborted.
 	Cancel()
+	// String returns the original query string.
+	String() string
 }
 
 // query implements the Query interface.
@@ -141,10 +143,17 @@ type query struct {
 type QueryOrigin struct{}
 
 // Statement implements the Query interface.
+// Calling this after Exec may result in panic,
+// see https://github.com/prometheus/prometheus/issues/8949.
 func (q *query) Statement() parser.Statement {
 	return q.stmt
 }
 
+// String implements the Query interface.
+func (q *query) String() string {
+	return q.q
+}
+
 // Stats implements the Query interface.
 func (q *query) Stats() *stats.QueryTimers {
 	return q.stats
@@ -521,8 +530,6 @@ func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storag
 	// Cancel when execution is done or an error was raised.
 	defer q.cancel()
 
-	const env = "query execution"
-
 	evalSpanTimer, ctx := q.stats.GetSpanTimer(ctx, stats.EvalTotalTime)
 	defer evalSpanTimer.Finish()
 

promql/engine_test.go

@@ -184,8 +184,10 @@ func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) stor
 func (*errQuerier) LabelValues(string, ...*labels.Matcher) ([]string, storage.Warnings, error) {
 	return nil, nil, nil
 }
-func (*errQuerier) LabelNames() ([]string, storage.Warnings, error) { return nil, nil, nil }
+func (*errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) {
-func (*errQuerier) Close() error                                    { return nil }
+	return nil, nil, nil
+}
+func (*errQuerier) Close() error { return nil }
 
 // errSeriesSet implements storage.SeriesSet which always returns error.
 type errSeriesSet struct {

promql/functions.go

@@ -513,6 +513,13 @@ func funcAbsentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalN
 		})
 }
 
+// === present_over_time(Vector parser.ValueTypeMatrix) Vector ===
+func funcPresentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector {
+	return aggrOverTime(vals, enh, func(values []Point) float64 {
+		return 1
+	})
+}
+
 func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float64) Vector {
 	for _, el := range vals[0].(Vector) {
 		enh.Out = append(enh.Out, Sample{
@@ -959,6 +966,7 @@ var FunctionCalls = map[string]FunctionCall{
 	"minute":             funcMinute,
 	"month":              funcMonth,
 	"predict_linear":     funcPredictLinear,
+	"present_over_time":  funcPresentOverTime,
 	"quantile_over_time": funcQuantileOverTime,
 	"rate":               funcRate,
 	"resets":             funcResets,

promql/parser/functions.go

@@ -39,6 +39,11 @@ var Functions = map[string]*Function{
 		ArgTypes:   []ValueType{ValueTypeMatrix},
 		ReturnType: ValueTypeVector,
 	},
+	"present_over_time": {
+		Name:       "present_over_time",
+		ArgTypes:   []ValueType{ValueTypeMatrix},
+		ReturnType: ValueTypeVector,
+	},
 	"avg_over_time": {
 		Name:       "avg_over_time",
 		ArgTypes:   []ValueType{ValueTypeMatrix},

promql/test.go

@@ -583,7 +583,7 @@ func (t *Test) exec(tc testCommand) error {
 				err = cmd.compareResult(vec)
 			}
 			if err != nil {
-				return errors.Wrapf(err, "error in %s %s (line %d) rande mode", cmd, iq.expr, cmd.line)
+				return errors.Wrapf(err, "error in %s %s (line %d) range mode", cmd, iq.expr, cmd.line)
 			}
 
 		}
@@ -675,12 +675,21 @@ type LazyLoader struct {
 	queryEngine *Engine
 	context     context.Context
 	cancelCtx   context.CancelFunc
+
+	opts LazyLoaderOpts
+}
+
+// LazyLoaderOpts are options for the lazy loader.
+type LazyLoaderOpts struct {
+	// Disabled PromQL engine features.
+	EnableAtModifier, EnableNegativeOffset bool
 }
 
 // NewLazyLoader returns an initialized empty LazyLoader.
-func NewLazyLoader(t testutil.T, input string) (*LazyLoader, error) {
+func NewLazyLoader(t testutil.T, input string, opts LazyLoaderOpts) (*LazyLoader, error) {
 	ll := &LazyLoader{
-		T: t,
+		T:    t,
+		opts: opts,
 	}
 	err := ll.parse(input)
 	ll.clear()
@@ -728,7 +737,8 @@ func (ll *LazyLoader) clear() {
 		MaxSamples:               10000,
 		Timeout:                  100 * time.Second,
 		NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(ll.SubqueryInterval) },
-		EnableAtModifier:         true,
+		EnableAtModifier:         ll.opts.EnableAtModifier,
+		EnableNegativeOffset:     ll.opts.EnableNegativeOffset,
 	}
 
 	ll.queryEngine = NewEngine(opts)

promql/test_test.go

@@ -109,7 +109,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		suite, err := NewLazyLoader(t, c.loadString)
+		suite, err := NewLazyLoader(t, c.loadString, LazyLoaderOpts{})
 		require.NoError(t, err)
 		defer suite.Close()
 

promql/testdata/functions.test

@@ -901,6 +901,66 @@ eval instant at 10m absent_over_time({job="ingress"}[4m])
 
 clear
 
+# Testdata for present_over_time()
+eval instant at 1m present_over_time(http_requests[5m])
+
+eval instant at 1m present_over_time(http_requests{handler="/foo"}[5m])
+
+eval instant at 1m present_over_time(http_requests{handler!="/foo"}[5m])
+
+eval instant at 1m present_over_time(http_requests{handler="/foo", handler="/bar", handler="/foobar"}[5m])
+
+eval instant at 1m present_over_time(rate(nonexistant[5m])[5m:])
+
+eval instant at 1m present_over_time(http_requests{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m])
+
+load 1m
+	http_requests{path="/foo",instance="127.0.0.1",job="httpd"}	1+1x10
+	http_requests{path="/bar",instance="127.0.0.1",job="httpd"}	1+1x10
+	httpd_handshake_failures_total{instance="127.0.0.1",job="node"}	1+1x15
+	httpd_log_lines_total{instance="127.0.0.1",job="node"}	1
+	ssl_certificate_expiry_seconds{job="ingress"} NaN NaN NaN NaN NaN
+
+eval instant at 5m present_over_time(http_requests[5m])
+    {instance="127.0.0.1", job="httpd", path="/bar"} 1
+    {instance="127.0.0.1", job="httpd", path="/foo"} 1
+
+eval instant at 5m present_over_time(rate(http_requests[5m])[5m:1m])
+    {instance="127.0.0.1", job="httpd", path="/bar"} 1
+    {instance="127.0.0.1", job="httpd", path="/foo"} 1
+
+eval instant at 0m present_over_time(httpd_log_lines_total[30s])
+    {instance="127.0.0.1",job="node"} 1
+
+eval instant at 1m present_over_time(httpd_log_lines_total[30s])
+
+eval instant at 15m present_over_time(http_requests[5m])
+    {instance="127.0.0.1", job="httpd", path="/bar"} 1
+    {instance="127.0.0.1", job="httpd", path="/foo"} 1
+
+eval instant at 16m present_over_time(http_requests[5m])
+
+eval instant at 16m present_over_time(http_requests[6m])
+    {instance="127.0.0.1", job="httpd", path="/bar"} 1
+    {instance="127.0.0.1", job="httpd", path="/foo"} 1
+
+eval instant at 16m present_over_time(httpd_handshake_failures_total[1m])
+    {instance="127.0.0.1", job="node"} 1
+
+eval instant at 16m present_over_time({instance="127.0.0.1"}[5m])
+    {instance="127.0.0.1",job="node"} 1
+
+eval instant at 21m present_over_time({job="grok"}[20m])
+
+eval instant at 30m present_over_time({instance="127.0.0.1"}[5m:5s])
+
+eval instant at 5m present_over_time({job="ingress"}[4m])
+    {job="ingress"} 1
+
+eval instant at 10m present_over_time({job="ingress"}[4m])
+
+clear
+
 # Testing exp() sqrt() log2() log10() ln()
 load 5m
 	exp_root_log{l="x"} 10

promql/value.go

@@ -171,8 +171,8 @@ func (m Matrix) Len() int           { return len(m) }
 func (m Matrix) Less(i, j int) bool { return labels.Compare(m[i].Metric, m[j].Metric) < 0 }
 func (m Matrix) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
 
-// ContainsSameLabelset checks if a matrix has samples with the same labelset
+// ContainsSameLabelset checks if a matrix has samples with the same labelset.
-// Such a behavior is semantically undefined
+// Such a behavior is semantically undefined.
 // https://github.com/prometheus/prometheus/issues/4562
 func (m Matrix) ContainsSameLabelset() bool {
 	l := make(map[uint64]struct{}, len(m))

scrape/manager.go

@@ -14,11 +14,8 @@
 package scrape
 
 import (
-	"encoding"
 	"fmt"
 	"hash/fnv"
-	"net"
-	"os"
 	"reflect"
 	"sync"
 	"time"
@@ -32,6 +29,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/osutil"
 )
 
 var targetMetadataCache = newMetadataMetricsCollector()
@@ -119,7 +117,7 @@ func NewManager(logger log.Logger, app storage.Appendable) *Manager {
 }
 
 // Manager maintains a set of scrape pools and manages start/stop cycles
-// when receiving new target groups form the discovery manager.
+// when receiving new target groups from the discovery manager.
 type Manager struct {
 	logger    log.Logger
 	append    storage.Appendable
@@ -206,7 +204,7 @@ func (m *Manager) reload() {
 // setJitterSeed calculates a global jitterSeed per server relying on extra label set.
 func (m *Manager) setJitterSeed(labels labels.Labels) error {
 	h := fnv.New64a()
-	hostname, err := getFqdn()
+	hostname, err := osutil.GetFQDN()
 	if err != nil {
 		return err
 	}
@@ -319,46 +317,3 @@ func (m *Manager) TargetsDropped() map[string][]*Target {
 	}
 	return targets
 }
-
-// getFqdn returns a FQDN if it's possible, otherwise falls back to hostname.
-func getFqdn() (string, error) {
-	hostname, err := os.Hostname()
-	if err != nil {
-		return "", err
-	}
-
-	ips, err := net.LookupIP(hostname)
-	if err != nil {
-		// Return the system hostname if we can't look up the IP address.
-		return hostname, nil
-	}
-
-	lookup := func(ipStr encoding.TextMarshaler) (string, error) {
-		ip, err := ipStr.MarshalText()
-		if err != nil {
-			return "", err
-		}
-		hosts, err := net.LookupAddr(string(ip))
-		if err != nil || len(hosts) == 0 {
-			return "", err
-		}
-		return hosts[0], nil
-	}
-
-	for _, addr := range ips {
-		if ip := addr.To4(); ip != nil {
-			if fqdn, err := lookup(ip); err == nil {
-				return fqdn, nil
-			}
-
-		}
-
-		if ip := addr.To16(); ip != nil {
-			if fqdn, err := lookup(ip); err == nil {
-				return fqdn, nil
-			}
-
-		}
-	}
-	return hostname, nil
-}

scrape/scrape.go

@@ -226,11 +226,10 @@ type scrapePool struct {
 	cancel     context.CancelFunc
 
 	// mtx must not be taken after targetMtx.
-	mtx            sync.Mutex
+	mtx    sync.Mutex
-	config         *config.ScrapeConfig
+	config *config.ScrapeConfig
-	client         *http.Client
+	client *http.Client
-	loops          map[uint64]loop
+	loops  map[uint64]loop
-	targetLimitHit bool // Internal state to speed up the target_limit checks.
 
 	targetMtx sync.Mutex
 	// activeTargets and loops must always be synchronized to have the same
@@ -575,20 +574,14 @@ func (sp *scrapePool) sync(targets []*Target) {
 // refreshTargetLimitErr returns an error that can be passed to the scrape loops
 // if the number of targets exceeds the configured limit.
 func (sp *scrapePool) refreshTargetLimitErr() error {
-	if sp.config == nil || sp.config.TargetLimit == 0 && !sp.targetLimitHit {
+	if sp.config == nil || sp.config.TargetLimit == 0 {
 		return nil
 	}
-	l := len(sp.activeTargets)
+	if l := len(sp.activeTargets); l > int(sp.config.TargetLimit) {
-	if l <= int(sp.config.TargetLimit) && !sp.targetLimitHit {
-		return nil
-	}
-	var err error
-	sp.targetLimitHit = l > int(sp.config.TargetLimit)
-	if sp.targetLimitHit {
 		targetScrapePoolExceededTargetLimit.Inc()
-		err = fmt.Errorf("target_limit exceeded (number of targets: %d, limit: %d)", l, sp.config.TargetLimit)
+		return fmt.Errorf("target_limit exceeded (number of targets: %d, limit: %d)", l, sp.config.TargetLimit)
 	}
-	return err
+	return nil
 }
 
 func verifyLabelLimits(lset labels.Labels, limits *labelLimits) error {

scrape/scrape_test.go

@@ -423,6 +423,10 @@ func TestScrapePoolTargetLimit(t *testing.T) {
 	validateIsRunning()
 	validateErrorMessage(true)
 
+	reloadWithLimit(0)
+	validateIsRunning()
+	validateErrorMessage(false)
+
 	reloadWithLimit(51)
 	validateIsRunning()
 	validateErrorMessage(false)

storage/fanout_test.go

@@ -234,7 +234,7 @@ func (errQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]strin
 	return nil, nil, errors.New("label values error")
 }
 
-func (errQuerier) LabelNames() ([]string, storage.Warnings, error) {
+func (errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) {
 	return nil, nil, errors.New("label names error")
 }
 

storage/interface.go

@@ -34,6 +34,7 @@ var (
 	ErrOutOfOrderExemplar          = errors.New("out of order exemplar")
 	ErrDuplicateExemplar           = errors.New("duplicate exemplar")
 	ErrExemplarLabelLength         = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength)
+	ErrExemplarsDisabled           = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0")
 )
 
 // Appendable allows creating appenders.
@@ -113,8 +114,9 @@ type LabelQuerier interface {
 	LabelValues(name string, matchers ...*labels.Matcher) ([]string, Warnings, error)
 
 	// LabelNames returns all the unique label names present in the block in sorted order.
-	// TODO(yeya24): support matchers or hints.
+	// If matchers are specified the returned result set is reduced
-	LabelNames() ([]string, Warnings, error)
+	// to label names of metrics matching the matchers.
+	LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error)
 
 	// Close releases the resources of the Querier.
 	Close() error

storage/merge.go

@@ -219,13 +219,13 @@ func mergeStrings(a, b []string) []string {
 }
 
 // LabelNames returns all the unique label names present in all queriers in sorted order.
-func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) {
+func (q *mergeGenericQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) {
 	var (
 		labelNamesMap = make(map[string]struct{})
 		warnings      Warnings
 	)
 	for _, querier := range q.queriers {
-		names, wrn, err := querier.LabelNames()
+		names, wrn, err := querier.LabelNames(matchers...)
 		if wrn != nil {
 			// TODO(bwplotka): We could potentially wrap warnings.
 			warnings = append(warnings, wrn...)

storage/merge_test.go

@@ -778,7 +778,7 @@ func (m *mockGenericQuerier) LabelValues(name string, matchers ...*labels.Matche
 	return m.resp, m.warnings, m.err
 }
 
-func (m *mockGenericQuerier) LabelNames() ([]string, Warnings, error) {
+func (m *mockGenericQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) {
 	m.mtx.Lock()
 	m.labelNamesCalls++
 	m.mtx.Unlock()

storage/noop.go

@@ -32,7 +32,7 @@ func (noopQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warnings,
 	return nil, nil, nil
 }
 
-func (noopQuerier) LabelNames() ([]string, Warnings, error) {
+func (noopQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) {
 	return nil, nil, nil
 }
 
@@ -55,7 +55,7 @@ func (noopChunkQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warni
 	return nil, nil, nil
 }
 
-func (noopChunkQuerier) LabelNames() ([]string, Warnings, error) {
+func (noopChunkQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) {
 	return nil, nil, nil
 }
 

storage/remote/queue_manager.go

@@ -352,10 +352,12 @@ type QueueManager struct {
 	clientMtx   sync.RWMutex
 	storeClient WriteClient
 
-	seriesMtx            sync.Mutex
+	seriesMtx     sync.Mutex // Covers seriesLabels and droppedSeries.
-	seriesLabels         map[uint64]labels.Labels
+	seriesLabels  map[uint64]labels.Labels
+	droppedSeries map[uint64]struct{}
+
+	seriesSegmentMtx     sync.Mutex // Covers seriesSegmentIndexes - if you also lock seriesMtx, take seriesMtx first.
 	seriesSegmentIndexes map[uint64]int
-	droppedSeries        map[uint64]struct{}
 
 	shards      *shards
 	numShards   int
@@ -642,6 +644,8 @@ func (t *QueueManager) Stop() {
 func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) {
 	t.seriesMtx.Lock()
 	defer t.seriesMtx.Unlock()
+	t.seriesSegmentMtx.Lock()
+	defer t.seriesSegmentMtx.Unlock()
 	for _, s := range series {
 		// Just make sure all the Refs of Series will insert into seriesSegmentIndexes map for tracking.
 		t.seriesSegmentIndexes[s.Ref] = index
@@ -664,12 +668,23 @@ func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) {
 	}
 }
 
+// Update the segment number held against the series, so we can trim older ones in SeriesReset.
+func (t *QueueManager) UpdateSeriesSegment(series []record.RefSeries, index int) {
+	t.seriesSegmentMtx.Lock()
+	defer t.seriesSegmentMtx.Unlock()
+	for _, s := range series {
+		t.seriesSegmentIndexes[s.Ref] = index
+	}
+}
+
 // SeriesReset is used when reading a checkpoint. WAL Watcher should have
 // stored series records with the checkpoints index number, so we can now
 // delete any ref ID's lower than that # from the two maps.
 func (t *QueueManager) SeriesReset(index int) {
 	t.seriesMtx.Lock()
 	defer t.seriesMtx.Unlock()
+	t.seriesSegmentMtx.Lock()
+	defer t.seriesSegmentMtx.Unlock()
 	// Check for series that are in segments older than the checkpoint
 	// that were not also present in the checkpoint.
 	for k, v := range t.seriesSegmentIndexes {

storage/remote/queue_manager_test.go

@@ -295,10 +295,10 @@ func TestShutdown(t *testing.T) {
 	// be at least equal to deadline, otherwise the flush deadline
 	// was not respected.
 	duration := time.Since(start)
-	if duration > time.Duration(deadline+(deadline/10)) {
+	if duration > deadline+(deadline/10) {
 		t.Errorf("Took too long to shutdown: %s > %s", duration, deadline)
 	}
-	if duration < time.Duration(deadline) {
+	if duration < deadline {
 		t.Errorf("Shutdown occurred before flush deadline: %s < %s", duration, deadline)
 	}
 }
@@ -474,7 +474,7 @@ func TestShouldReshard(t *testing.T) {
 	}
 }
 
-func createTimeseries(numSamples, numSeries int) ([]record.RefSample, []record.RefSeries) {
+func createTimeseries(numSamples, numSeries int, extraLabels ...labels.Label) ([]record.RefSample, []record.RefSeries) {
 	samples := make([]record.RefSample, 0, numSamples)
 	series := make([]record.RefSeries, 0, numSeries)
 	for i := 0; i < numSeries; i++ {
@@ -488,7 +488,7 @@ func createTimeseries(numSamples, numSeries int) ([]record.RefSample, []record.R
 		}
 		series = append(series, record.RefSeries{
 			Ref:    uint64(i),
-			Labels: labels.Labels{{Name: "__name__", Value: name}},
+			Labels: append(labels.Labels{{Name: "__name__", Value: name}}, extraLabels...),
 		})
 	}
 	return samples, series
@@ -709,10 +709,29 @@ func (c *TestBlockingWriteClient) Endpoint() string {
 }
 
 func BenchmarkSampleDelivery(b *testing.B) {
-	// Let's create an even number of send batches so we don't run into the
+	// Send one sample per series, which is the typical remote_write case
-	// batch timeout case.
+	const numSamples = 1
-	n := config.DefaultQueueConfig.MaxSamplesPerSend * 10
+	const numSeries = 10000
-	samples, series := createTimeseries(n, n)
+
+	// Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics.
+	var extraLabels = labels.Labels{
+		{Name: "kubernetes_io_arch", Value: "amd64"},
+		{Name: "kubernetes_io_instance_type", Value: "c3.somesize"},
+		{Name: "kubernetes_io_os", Value: "linux"},
+		{Name: "container_name", Value: "some-name"},
+		{Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"},
+		{Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"},
+		{Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"},
+		{Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"},
+		{Name: "instance", Value: "ip-111-11-1-11.ec2.internal"},
+		{Name: "job", Value: "kubernetes-cadvisor"},
+		{Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"},
+		{Name: "monitor", Value: "prod"},
+		{Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"},
+		{Name: "namespace", Value: "kube-system"},
+		{Name: "pod_name", Value: "some-other-name-5j8s8"},
+	}
+	samples, series := createTimeseries(numSamples, numSeries, extraLabels...)
 
 	c := NewTestWriteClient()
 
@@ -736,7 +755,9 @@ func BenchmarkSampleDelivery(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		c.expectDataCount(len(samples))
-		m.Append(samples)
+		go m.Append(samples)
+		m.UpdateSeriesSegment(series, i+1) // simulate what wal.Watcher.garbageCollectSeries does
+		m.SeriesReset(i + 1)
 		c.waitForExpectedDataCount()
 	}
 	// Do not include shutdown

storage/remote/read.go

@@ -212,7 +212,7 @@ func (q *querier) LabelValues(string, ...*labels.Matcher) ([]string, storage.War
 }
 
 // LabelNames implements storage.Querier and is a noop.
-func (q *querier) LabelNames() ([]string, storage.Warnings, error) {
+func (q *querier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) {
 	// TODO: Implement: https://github.com/prometheus/prometheus/issues/3351
 	return nil, nil, errors.New("not implemented")
 }

storage/secondary.go

@@ -55,8 +55,8 @@ func (s *secondaryQuerier) LabelValues(name string, matchers ...*labels.Matcher)
 	return vals, w, nil
 }
 
-func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) {
+func (s *secondaryQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) {
-	names, w, err := s.genericQuerier.LabelNames()
+	names, w, err := s.genericQuerier.LabelNames(matchers...)
 	if err != nil {
 		return nil, append([]error{err}, w...), nil
 	}

tsdb/block.go

@@ -85,13 +85,17 @@ type IndexReader interface {
 	Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error
 
 	// LabelNames returns all the unique label names present in the index in sorted order.
-	LabelNames() ([]string, error)
+	LabelNames(matchers ...*labels.Matcher) ([]string, error)
 
 	// LabelValueFor returns label value for the given label name in the series referred to by ID.
 	// If the series couldn't be found or the series doesn't have the requested label a
 	// storage.ErrNotFound is returned as error.
 	LabelValueFor(id uint64, label string) (string, error)
 
+	// LabelNamesFor returns all the label names for the series referred to by IDs.
+	// The names returned are sorted.
+	LabelNamesFor(ids ...uint64) ([]string, error)
+
 	// Close releases the underlying resources of the reader.
 	Close() error
 }
@@ -443,7 +447,15 @@ func (r blockIndexReader) LabelValues(name string, matchers ...*labels.Matcher)
 		return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
 	}
 
-	return labelValuesWithMatchers(r, name, matchers...)
+	return labelValuesWithMatchers(r.ir, name, matchers...)
+}
+
+func (r blockIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) {
+	if len(matchers) == 0 {
+		return r.b.LabelNames()
+	}
+
+	return labelNamesWithMatchers(r.ir, matchers...)
 }
 
 func (r blockIndexReader) Postings(name string, values ...string) (index.Postings, error) {
@@ -465,10 +477,6 @@ func (r blockIndexReader) Series(ref uint64, lset *labels.Labels, chks *[]chunks
 	return nil
 }
 
-func (r blockIndexReader) LabelNames() ([]string, error) {
-	return r.b.LabelNames()
-}
-
 func (r blockIndexReader) Close() error {
 	r.b.pendingReaders.Done()
 	return nil
@@ -479,6 +487,12 @@ func (r blockIndexReader) LabelValueFor(id uint64, label string) (string, error)
 	return r.ir.LabelValueFor(id, label)
 }
 
+// LabelNamesFor returns all the label names for the series referred to by IDs.
+// The names returned are sorted.
+func (r blockIndexReader) LabelNamesFor(ids ...uint64) ([]string, error) {
+	return r.ir.LabelNamesFor(ids...)
+}
+
 type blockTombstoneReader struct {
 	tombstones.Reader
 	b *Block

tsdb/block_test.go

@@ -418,6 +418,82 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) {
 	}
 }
 
+func TestLabelNamesWithMatchers(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "test_block_label_names_with_matchers")
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpdir)) })
+
+	var seriesEntries []storage.Series
+	for i := 0; i < 100; i++ {
+		seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{
+			{Name: "unique", Value: fmt.Sprintf("value%d", i)},
+		}, []tsdbutil.Sample{sample{100, 0}}))
+
+		if i%10 == 0 {
+			seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{
+				{Name: "unique", Value: fmt.Sprintf("value%d", i)},
+				{Name: "tens", Value: fmt.Sprintf("value%d", i/10)},
+			}, []tsdbutil.Sample{sample{100, 0}}))
+		}
+
+		if i%20 == 0 {
+			seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{
+				{Name: "unique", Value: fmt.Sprintf("value%d", i)},
+				{Name: "tens", Value: fmt.Sprintf("value%d", i/10)},
+				{Name: "twenties", Value: fmt.Sprintf("value%d", i/20)},
+			}, []tsdbutil.Sample{sample{100, 0}}))
+		}
+
+	}
+
+	blockDir := createBlock(t, tmpdir, seriesEntries)
+	files, err := sequenceFiles(chunkDir(blockDir))
+	require.NoError(t, err)
+	require.Greater(t, len(files), 0, "No chunk created.")
+
+	// Check open err.
+	block, err := OpenBlock(nil, blockDir, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, block.Close()) })
+
+	indexReader, err := block.Index()
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, indexReader.Close()) })
+
+	testCases := []struct {
+		name          string
+		labelName     string
+		matchers      []*labels.Matcher
+		expectedNames []string
+	}{
+		{
+			name:          "get with non-empty unique: all",
+			matchers:      []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")},
+			expectedNames: []string{"tens", "twenties", "unique"},
+		}, {
+			name:          "get with unique ending in 1: only unique",
+			matchers:      []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value.*1")},
+			expectedNames: []string{"unique"},
+		}, {
+			name:          "get with unique = value20: all",
+			matchers:      []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "unique", "value20")},
+			expectedNames: []string{"tens", "twenties", "unique"},
+		}, {
+			name:          "get tens = 1: unique & tens",
+			matchers:      []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "tens", "value1")},
+			expectedNames: []string{"tens", "unique"},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			actualNames, err := indexReader.LabelNames(tt.matchers...)
+			require.NoError(t, err)
+			require.Equal(t, tt.expectedNames, actualNames)
+		})
+	}
+}
+
 // createBlock creates a block with given set of series and returns its dir.
 func createBlock(tb testing.TB, dir string, series []storage.Series) string {
 	blockDir, err := CreateBlock(series, dir, 0, log.NewNopLogger())

tsdb/chunkenc/bstream.go

@@ -95,7 +95,7 @@ func (b *bstream) writeByte(byt byte) {
 }
 
 func (b *bstream) writeBits(u uint64, nbits int) {
-	u <<= (64 - uint(nbits))
+	u <<= 64 - uint(nbits)
 	for nbits >= 8 {
 		byt := byte(u >> 56)
 		b.writeByte(byt)

tsdb/chunkenc/chunk_test.go

@@ -110,12 +110,13 @@ func testChunk(t *testing.T, c Chunk) {
 }
 
 func benchmarkIterator(b *testing.B, newChunk func() Chunk) {
+	const samplesPerChunk = 250
 	var (
 		t   = int64(1234123324)
 		v   = 1243535.123
 		exp []pair
 	)
-	for i := 0; i < b.N; i++ {
+	for i := 0; i < samplesPerChunk; i++ {
 		// t += int64(rand.Intn(10000) + 1)
 		t += int64(1000)
 		// v = rand.Float64()
@@ -123,11 +124,9 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) {
 		exp = append(exp, pair{t: t, v: v})
 	}
 
-	var chunks []Chunk
+	chunk := newChunk()
-	for i := 0; i < b.N; {
+	{
-		c := newChunk()
+		a, err := chunk.Appender()
-
-		a, err := c.Appender()
 		if err != nil {
 			b.Fatalf("get appender: %s", err)
 		}
@@ -137,32 +136,27 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) {
 				break
 			}
 			a.Append(p.t, p.v)
-			i++
 			j++
 		}
-		chunks = append(chunks, c)
 	}
 
 	b.ReportAllocs()
 	b.ResetTimer()
 
-	b.Log("num", b.N, "created chunks", len(chunks))
+	var res float64
-
-	res := make([]float64, 0, 1024)
-
 	var it Iterator
-	for i := 0; i < len(chunks); i++ {
+	for i := 0; i < b.N; {
-		c := chunks[i]
+		it := chunk.Iterator(it)
-		it := c.Iterator(it)
 
 		for it.Next() {
 			_, v := it.At()
-			res = append(res, v)
+			res = v
+			i++
 		}
 		if it.Err() != io.EOF {
 			require.NoError(b, it.Err())
 		}
-		res = res[:0]
+		_ = res
 	}
 }
 

tsdb/chunkenc/xor.go

@@ -185,16 +185,16 @@ func (a *xorAppender) Append(t int64, v float64) {
 		case dod == 0:
 			a.b.writeBit(zero)
 		case bitRange(dod, 14):
-			a.b.writeBits(0x02, 2) // '10'
+			a.b.writeBits(0b10, 2)
 			a.b.writeBits(uint64(dod), 14)
 		case bitRange(dod, 17):
-			a.b.writeBits(0x06, 3) // '110'
+			a.b.writeBits(0b110, 3)
 			a.b.writeBits(uint64(dod), 17)
 		case bitRange(dod, 20):
-			a.b.writeBits(0x0e, 4) // '1110'
+			a.b.writeBits(0b1110, 4)
 			a.b.writeBits(uint64(dod), 20)
 		default:
-			a.b.writeBits(0x0f, 4) // '1111'
+			a.b.writeBits(0b1111, 4)
 			a.b.writeBits(uint64(dod), 64)
 		}
 
@@ -359,15 +359,15 @@ func (it *xorIterator) Next() bool {
 	var sz uint8
 	var dod int64
 	switch d {
-	case 0x00:
+	case 0b0:
 		// dod == 0
-	case 0x02:
+	case 0b10:
 		sz = 14
-	case 0x06:
+	case 0b110:
 		sz = 17
-	case 0x0e:
+	case 0b1110:
 		sz = 20
-	case 0x0f:
+	case 0b1111:
 		// Do not use fast because it's very unlikely it will succeed.
 		bits, err := it.br.readBits(64)
 		if err != nil {

tsdb/chunks/head_chunks.go

@@ -386,7 +386,7 @@ func (cdm *ChunkDiskMapper) cut() (returnErr error) {
 		cdm.readPathMtx.Unlock()
 	}
 
-	mmapFile, err := fileutil.OpenMmapFileWithSize(newFile.Name(), int(MaxHeadChunkFileSize))
+	mmapFile, err := fileutil.OpenMmapFileWithSize(newFile.Name(), MaxHeadChunkFileSize)
 	if err != nil {
 		return err
 	}

tsdb/db.go

@@ -35,6 +35,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -147,9 +148,12 @@ type Options struct {
 	// mainly meant for external users who import TSDB.
 	BlocksToDelete BlocksToDeleteFunc
 
+	// Enables the in memory exemplar storage,.
+	EnableExemplarStorage bool
+
 	// MaxExemplars sets the size, in # of exemplars stored, of the single circular buffer used to store exemplars in memory.
 	// See tsdb/exemplar.go, specifically the CircularExemplarStorage struct and it's constructor NewCircularExemplarStorage.
-	MaxExemplars int
+	MaxExemplars int64
 }
 
 type BlocksToDeleteFunc func(blocks []*Block) map[ulid.ULID]struct{}
@@ -716,7 +720,8 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 	headOpts.ChunkWriteBufferSize = opts.HeadChunksWriteBufferSize
 	headOpts.StripeSize = opts.StripeSize
 	headOpts.SeriesCallback = opts.SeriesLifecycleCallback
-	headOpts.NumExemplars = opts.MaxExemplars
+	headOpts.EnableExemplarStorage = opts.EnableExemplarStorage
+	headOpts.MaxExemplars.Store(opts.MaxExemplars)
 	db.head, err = NewHead(r, l, wlog, headOpts, stats.Head)
 	if err != nil {
 		return nil, err
@@ -838,6 +843,10 @@ func (db *DB) Appender(ctx context.Context) storage.Appender {
 	return dbAppender{db: db, Appender: db.head.Appender(ctx)}
 }
 
+func (db *DB) ApplyConfig(conf *config.Config) error {
+	return db.head.ApplyConfig(conf)
+}
+
 // dbAppender wraps the DB's head appender and triggers compactions on commit
 // if necessary.
 type dbAppender struct {
@@ -1516,8 +1525,32 @@ func (db *DB) Querier(_ context.Context, mint, maxt int64) (storage.Querier, err
 			blocks = append(blocks, b)
 		}
 	}
+	var headQuerier storage.Querier
 	if maxt >= db.head.MinTime() {
-		blocks = append(blocks, NewRangeHead(db.head, mint, maxt))
+		rh := NewRangeHead(db.head, mint, maxt)
+		var err error
+		headQuerier, err = NewBlockQuerier(rh, mint, maxt)
+		if err != nil {
+			return nil, errors.Wrapf(err, "open querier for head %s", rh)
+		}
+
+		// Getting the querier above registers itself in the queue that the truncation waits on.
+		// So if the querier is currently not colliding with any truncation, we can continue to use it and still
+		// won't run into a race later since any truncation that comes after will wait on this querier if it overlaps.
+		shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt)
+		if shouldClose {
+			if err := headQuerier.Close(); err != nil {
+				return nil, errors.Wrapf(err, "closing head querier %s", rh)
+			}
+			headQuerier = nil
+		}
+		if getNew {
+			rh := NewRangeHead(db.head, newMint, maxt)
+			headQuerier, err = NewBlockQuerier(rh, newMint, maxt)
+			if err != nil {
+				return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh)
+			}
+		}
 	}
 
 	blockQueriers := make([]storage.Querier, 0, len(blocks))
@@ -1534,6 +1567,9 @@ func (db *DB) Querier(_ context.Context, mint, maxt int64) (storage.Querier, err
 		}
 		return nil, errors.Wrapf(err, "open querier for block %s", b)
 	}
+	if headQuerier != nil {
+		blockQueriers = append(blockQueriers, headQuerier)
+	}
 	return storage.NewMergeQuerier(blockQueriers, nil, storage.ChainedSeriesMerge), nil
 }
 
@@ -1549,8 +1585,32 @@ func (db *DB) ChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQu
 			blocks = append(blocks, b)
 		}
 	}
+	var headQuerier storage.ChunkQuerier
 	if maxt >= db.head.MinTime() {
-		blocks = append(blocks, NewRangeHead(db.head, mint, maxt))
+		rh := NewRangeHead(db.head, mint, maxt)
+		var err error
+		headQuerier, err = NewBlockChunkQuerier(rh, mint, maxt)
+		if err != nil {
+			return nil, errors.Wrapf(err, "open querier for head %s", rh)
+		}
+
+		// Getting the querier above registers itself in the queue that the truncation waits on.
+		// So if the querier is currently not colliding with any truncation, we can continue to use it and still
+		// won't run into a race later since any truncation that comes after will wait on this querier if it overlaps.
+		shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt)
+		if shouldClose {
+			if err := headQuerier.Close(); err != nil {
+				return nil, errors.Wrapf(err, "closing head querier %s", rh)
+			}
+			headQuerier = nil
+		}
+		if getNew {
+			rh := NewRangeHead(db.head, newMint, maxt)
+			headQuerier, err = NewBlockChunkQuerier(rh, newMint, maxt)
+			if err != nil {
+				return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh)
+			}
+		}
 	}
 
 	blockQueriers := make([]storage.ChunkQuerier, 0, len(blocks))
@@ -1567,6 +1627,9 @@ func (db *DB) ChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQu
 		}
 		return nil, errors.Wrapf(err, "open querier for block %s", b)
 	}
+	if headQuerier != nil {
+		blockQueriers = append(blockQueriers, headQuerier)
+	}
 
 	return storage.NewMergeChunkQuerier(blockQueriers, nil, storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)), nil
 }

tsdb/db_test.go

@@ -3444,3 +3444,18 @@ func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChun
 		require.NoError(t, err)
 	}
 }
+
+func newTestDB(t *testing.T) *DB {
+	dir, err := ioutil.TempDir("", "test")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, os.RemoveAll(dir))
+	})
+
+	db, err := Open(dir, nil, nil, DefaultOptions(), nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, db.Close())
+	})
+	return db
+}

tsdb/encoding/encoding.go

@@ -19,6 +19,7 @@ import (
 	"hash/crc32"
 	"unsafe"
 
+	"github.com/dennwc/varint"
 	"github.com/pkg/errors"
 )
 
@@ -139,7 +140,7 @@ func NewDecbufUvarintAt(bs ByteSlice, off int, castagnoliTable *crc32.Table) Dec
 	}
 	b := bs.Range(off, off+binary.MaxVarintLen32)
 
-	l, n := binary.Uvarint(b)
+	l, n := varint.Uvarint(b)
 	if n <= 0 || n > binary.MaxVarintLen32 {
 		return Decbuf{E: errors.Errorf("invalid uvarint %d", n)}
 	}
@@ -207,11 +208,17 @@ func (d *Decbuf) Varint64() int64 {
 	if d.E != nil {
 		return 0
 	}
-	x, n := binary.Varint(d.B)
+	// Decode as unsigned first, since that's what the varint library implements.
+	ux, n := varint.Uvarint(d.B)
 	if n < 1 {
 		d.E = ErrInvalidSize
 		return 0
 	}
+	// Now decode "ZigZag encoding" https://developers.google.com/protocol-buffers/docs/encoding#signed_integers.
+	x := int64(ux >> 1)
+	if ux&1 != 0 {
+		x = ^x
+	}
 	d.B = d.B[n:]
 	return x
 }
@@ -220,7 +227,7 @@ func (d *Decbuf) Uvarint64() uint64 {
 	if d.E != nil {
 		return 0
 	}
-	x, n := binary.Uvarint(d.B)
+	x, n := varint.Uvarint(d.B)
 	if n < 1 {
 		d.E = ErrInvalidSize
 		return 0

tsdb/exemplar.go

@@ -20,21 +20,20 @@ import (
 	"unicode/utf8"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/pkg/exemplar"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
 )
 
-type CircularExemplarStorage struct {
+// Indicates that there is no index entry for an exmplar.
-	exemplarsAppended            prometheus.Counter
+const noExemplar = -1
-	exemplarsInStorage           prometheus.Gauge
-	seriesWithExemplarsInStorage prometheus.Gauge
-	lastExemplarsTs              prometheus.Gauge
-	outOfOrderExemplars          prometheus.Counter
 
+type CircularExemplarStorage struct {
 	lock      sync.RWMutex
 	exemplars []*circularBufferEntry
 	nextIndex int
+	metrics   *ExemplarMetrics
 
 	// Map of series labels as a string to index entry, which points to the first
 	// and last exemplar for the series in the exemplars circular buffer.
@@ -53,17 +52,17 @@ type circularBufferEntry struct {
 	ref      *indexEntry
 }
 
-// NewCircularExemplarStorage creates an circular in memory exemplar storage.
+type ExemplarMetrics struct {
-// If we assume the average case 95 bytes per exemplar we can fit 5651272 exemplars in
+	exemplarsAppended            prometheus.Counter
-// 1GB of extra memory, accounting for the fact that this is heap allocated space.
+	exemplarsInStorage           prometheus.Gauge
-// If len < 1, then the exemplar storage is disabled.
+	seriesWithExemplarsInStorage prometheus.Gauge
-func NewCircularExemplarStorage(len int, reg prometheus.Registerer) (ExemplarStorage, error) {
+	lastExemplarsTs              prometheus.Gauge
-	if len < 1 {
+	maxExemplars                 prometheus.Gauge
-		return &noopExemplarStorage{}, nil
+	outOfOrderExemplars          prometheus.Counter
-	}
+}
-	c := &CircularExemplarStorage{
+
-		exemplars: make([]*circularBufferEntry, len),
+func NewExemplarMetrics(reg prometheus.Registerer) *ExemplarMetrics {
-		index:     make(map[string]*indexEntry),
+	m := ExemplarMetrics{
 		exemplarsAppended: prometheus.NewCounter(prometheus.CounterOpts{
 			Name: "prometheus_tsdb_exemplar_exemplars_appended_total",
 			Help: "Total number of appended exemplars.",
@@ -86,20 +85,51 @@ func NewCircularExemplarStorage(len int, reg prometheus.Registerer) (ExemplarSto
 			Name: "prometheus_tsdb_exemplar_out_of_order_exemplars_total",
 			Help: "Total number of out of order exemplar ingestion failed attempts.",
 		}),
+		maxExemplars: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "prometheus_tsdb_exemplar_max_exemplars",
+			Help: "Total number of exemplars the exemplar storage can store, resizeable.",
+		}),
 	}
+
 	if reg != nil {
 		reg.MustRegister(
-			c.exemplarsAppended,
+			m.exemplarsAppended,
-			c.exemplarsInStorage,
+			m.exemplarsInStorage,
-			c.seriesWithExemplarsInStorage,
+			m.seriesWithExemplarsInStorage,
-			c.lastExemplarsTs,
+			m.lastExemplarsTs,
-			c.outOfOrderExemplars,
+			m.outOfOrderExemplars,
+			m.maxExemplars,
 		)
 	}
 
+	return &m
+}
+
+// NewCircularExemplarStorage creates an circular in memory exemplar storage.
+// If we assume the average case 95 bytes per exemplar we can fit 5651272 exemplars in
+// 1GB of extra memory, accounting for the fact that this is heap allocated space.
+// If len <= 0, then the exemplar storage is essentially a noop storage but can later be
+// resized to store exemplars.
+func NewCircularExemplarStorage(len int64, m *ExemplarMetrics) (ExemplarStorage, error) {
+	if len < 0 {
+		len = 0
+	}
+	c := &CircularExemplarStorage{
+		exemplars: make([]*circularBufferEntry, len),
+		index:     make(map[string]*indexEntry),
+		metrics:   m,
+	}
+
+	c.metrics.maxExemplars.Set(float64(len))
+
 	return c, nil
 }
 
+func (ce *CircularExemplarStorage) ApplyConfig(cfg *config.Config) error {
+	ce.Resize(cfg.StorageConfig.ExemplarsConfig.MaxExemplars)
+	return nil
+}
+
 func (ce *CircularExemplarStorage) Appender() *CircularExemplarStorage {
 	return ce
 }
@@ -116,6 +146,10 @@ func (ce *CircularExemplarStorage) Querier(_ context.Context) (storage.ExemplarQ
 func (ce *CircularExemplarStorage) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) {
 	ret := make([]exemplar.QueryResult, 0)
 
+	if len(ce.exemplars) <= 0 {
+		return ret, nil
+	}
+
 	ce.lock.RLock()
 	defer ce.lock.RUnlock()
 
@@ -136,7 +170,7 @@ func (ce *CircularExemplarStorage) Select(start, end int64, matchers ...[]*label
 			if e.exemplar.Ts >= start {
 				se.Exemplars = append(se.Exemplars, e.exemplar)
 			}
-			if e.next == -1 {
+			if e.next == noExemplar {
 				break
 			}
 			e = ce.exemplars[e.next]
@@ -179,6 +213,10 @@ func (ce *CircularExemplarStorage) ValidateExemplar(l labels.Labels, e exemplar.
 // Not thread safe. The append parameters tells us whether this is an external validation, or internal
 // as a result of an AddExemplar call, in which case we should update any relevant metrics.
 func (ce *CircularExemplarStorage) validateExemplar(l string, e exemplar.Exemplar, append bool) error {
+	if len(ce.exemplars) <= 0 {
+		return storage.ErrExemplarsDisabled
+	}
+
 	// Exemplar label length does not include chars involved in text rendering such as quotes
 	// equals sign, or commas. See definition of const ExemplarMaxLabelLength.
 	labelSetLen := 0
@@ -204,14 +242,92 @@ func (ce *CircularExemplarStorage) validateExemplar(l string, e exemplar.Exempla
 
 	if e.Ts <= ce.exemplars[idx.newest].exemplar.Ts {
 		if append {
-			ce.outOfOrderExemplars.Inc()
+			ce.metrics.outOfOrderExemplars.Inc()
 		}
 		return storage.ErrOutOfOrderExemplar
 	}
 	return nil
 }
 
+// Resize changes the size of exemplar buffer by allocating a new buffer and migrating data to it.
+// Exemplars are kept when possible. Shrinking will discard oldest data (in order of ingest) as needed.
+func (ce *CircularExemplarStorage) Resize(l int64) int {
+	// Accept negative values as just 0 size.
+	if l <= 0 {
+		l = 0
+	}
+
+	if l == int64(len(ce.exemplars)) {
+		return 0
+	}
+
+	ce.lock.Lock()
+	defer ce.lock.Unlock()
+
+	oldBuffer := ce.exemplars
+	oldNextIndex := int64(ce.nextIndex)
+
+	ce.exemplars = make([]*circularBufferEntry, l)
+	ce.index = make(map[string]*indexEntry)
+	ce.nextIndex = 0
+
+	// Replay as many entries as needed, starting with oldest first.
+	count := int64(len(oldBuffer))
+	if l < count {
+		count = l
+	}
+
+	migrated := 0
+
+	if l > 0 {
+		// Rewind previous next index by count with wrap-around.
+		// This math is essentially looking at nextIndex, where we would write the next exemplar to,
+		// and find the index in the old exemplar buffer that we should start migrating exemplars from.
+		// This way we don't migrate exemplars that would just be overwritten when migrating later exemplars.
+		var startIndex int64 = (oldNextIndex - count + int64(len(oldBuffer))) % int64(len(oldBuffer))
+
+		for i := int64(0); i < count; i++ {
+			idx := (startIndex + i) % int64(len(oldBuffer))
+			if entry := oldBuffer[idx]; entry != nil {
+				ce.migrate(entry)
+				migrated++
+			}
+		}
+	}
+
+	ce.computeMetrics()
+	ce.metrics.maxExemplars.Set(float64(l))
+
+	return migrated
+}
+
+// migrate is like AddExemplar but reuses existing structs. Expected to be called in batch and requires
+// external lock and does not compute metrics.
+func (ce *CircularExemplarStorage) migrate(entry *circularBufferEntry) {
+	seriesLabels := entry.ref.seriesLabels.String()
+
+	idx, ok := ce.index[seriesLabels]
+	if !ok {
+		idx = entry.ref
+		idx.oldest = ce.nextIndex
+		ce.index[seriesLabels] = idx
+	} else {
+		entry.ref = idx
+		ce.exemplars[idx.newest].next = ce.nextIndex
+	}
+	idx.newest = ce.nextIndex
+
+	entry.next = noExemplar
+	ce.exemplars[ce.nextIndex] = entry
+
+	ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars)
+}
+
 func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error {
+	if len(ce.exemplars) <= 0 {
+		return storage.ErrExemplarsDisabled
+	}
+
 	seriesLabels := l.String()
 
 	// TODO(bwplotka): This lock can lock all scrapers, there might high contention on this on scale.
@@ -241,7 +357,7 @@ func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemp
 		// There exists exemplar already on this ce.nextIndex entry, drop it, to make place
 		// for others.
 		prevLabels := prev.ref.seriesLabels.String()
-		if prev.next == -1 {
+		if prev.next == noExemplar {
 			// Last item for this series, remove index entry.
 			delete(ce.index, prevLabels)
 		} else {
@@ -251,43 +367,36 @@ func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemp
 
 	// Default the next value to -1 (which we use to detect that we've iterated through all exemplars for a series in Select)
 	// since this is the first exemplar stored for this series.
+	ce.exemplars[ce.nextIndex].next = noExemplar
 	ce.exemplars[ce.nextIndex].exemplar = e
-	ce.exemplars[ce.nextIndex].next = -1
 	ce.exemplars[ce.nextIndex].ref = ce.index[seriesLabels]
 	ce.index[seriesLabels].newest = ce.nextIndex
 
 	ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars)
 
-	ce.exemplarsAppended.Inc()
+	ce.metrics.exemplarsAppended.Inc()
-	ce.seriesWithExemplarsInStorage.Set(float64(len(ce.index)))
+	ce.computeMetrics()
+	return nil
+}
+
+func (ce *CircularExemplarStorage) computeMetrics() {
+	ce.metrics.seriesWithExemplarsInStorage.Set(float64(len(ce.index)))
+
+	if len(ce.exemplars) == 0 {
+		ce.metrics.exemplarsInStorage.Set(float64(0))
+		ce.metrics.lastExemplarsTs.Set(float64(0))
+		return
+	}
+
 	if next := ce.exemplars[ce.nextIndex]; next != nil {
-		ce.exemplarsInStorage.Set(float64(len(ce.exemplars)))
+		ce.metrics.exemplarsInStorage.Set(float64(len(ce.exemplars)))
-		ce.lastExemplarsTs.Set(float64(next.exemplar.Ts) / 1000)
+		ce.metrics.lastExemplarsTs.Set(float64(next.exemplar.Ts) / 1000)
-		return nil
+		return
 	}
 
 	// We did not yet fill the buffer.
-	ce.exemplarsInStorage.Set(float64(ce.nextIndex))
+	ce.metrics.exemplarsInStorage.Set(float64(ce.nextIndex))
-	ce.lastExemplarsTs.Set(float64(ce.exemplars[0].exemplar.Ts) / 1000)
+	if ce.exemplars[0] != nil {
-	return nil
+		ce.metrics.lastExemplarsTs.Set(float64(ce.exemplars[0].exemplar.Ts) / 1000)
-}
+	}
-
-type noopExemplarStorage struct{}
-
-func (noopExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error {
-	return nil
-}
-
-func (noopExemplarStorage) ValidateExemplar(l labels.Labels, e exemplar.Exemplar) error {
-	return nil
-}
-
-func (noopExemplarStorage) ExemplarQuerier(context.Context) (storage.ExemplarQuerier, error) {
-	return &noopExemplarQuerier{}, nil
-}
-
-type noopExemplarQuerier struct{}
-
-func (noopExemplarQuerier) Select(_, _ int64, _ ...[]*labels.Matcher) ([]exemplar.QueryResult, error) {
-	return nil, nil
 }

tsdb/exemplar_test.go

@@ -14,6 +14,9 @@
 package tsdb
 
 import (
+	"context"
+	"fmt"
+	"math"
 	"reflect"
 	"strconv"
 	"strings"
@@ -21,14 +24,17 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/pkg/exemplar"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
 )
 
+var eMetrics = NewExemplarMetrics(prometheus.DefaultRegisterer)
+
 // Tests the same exemplar cases as AddExemplar, but specifically the ValidateExemplar function so it can be relied on externally.
 func TestValidateExemplar(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(2, nil)
+	exs, err := NewCircularExemplarStorage(2, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -87,7 +93,7 @@ func TestValidateExemplar(t *testing.T) {
 }
 
 func TestAddExemplar(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(2, nil)
+	exs, err := NewCircularExemplarStorage(2, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -150,7 +156,7 @@ func TestStorageOverflow(t *testing.T) {
 	// Test that circular buffer index and assignment
 	// works properly, adding more exemplars than can
 	// be stored and then querying for them.
-	exs, err := NewCircularExemplarStorage(5, nil)
+	exs, err := NewCircularExemplarStorage(5, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -185,7 +191,7 @@ func TestStorageOverflow(t *testing.T) {
 }
 
 func TestSelectExemplar(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(5, nil)
+	exs, err := NewCircularExemplarStorage(5, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -216,7 +222,7 @@ func TestSelectExemplar(t *testing.T) {
 }
 
 func TestSelectExemplar_MultiSeries(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(5, nil)
+	exs, err := NewCircularExemplarStorage(5, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -273,8 +279,8 @@ func TestSelectExemplar_MultiSeries(t *testing.T) {
 }
 
 func TestSelectExemplar_TimeRange(t *testing.T) {
-	lenEs := 5
+	var lenEs int64 = 5
-	exs, err := NewCircularExemplarStorage(lenEs, nil)
+	exs, err := NewCircularExemplarStorage(lenEs, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -282,7 +288,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) {
 		{Name: "service", Value: "asdf"},
 	}
 
-	for i := 0; i < lenEs; i++ {
+	for i := 0; int64(i) < lenEs; i++ {
 		err := es.AddExemplar(l, exemplar.Exemplar{
 			Labels: labels.Labels{
 				labels.Label{
@@ -308,7 +314,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) {
 // Test to ensure that even though a series matches more than one matcher from the
 // query that it's exemplars are only included in the result a single time.
 func TestSelectExemplar_DuplicateSeries(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(4, nil)
+	exs, err := NewCircularExemplarStorage(4, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -349,7 +355,7 @@ func TestSelectExemplar_DuplicateSeries(t *testing.T) {
 }
 
 func TestIndexOverwrite(t *testing.T) {
-	exs, err := NewCircularExemplarStorage(2, nil)
+	exs, err := NewCircularExemplarStorage(2, eMetrics)
 	require.NoError(t, err)
 	es := exs.(*CircularExemplarStorage)
 
@@ -380,3 +386,162 @@ func TestIndexOverwrite(t *testing.T) {
 	i := es.index[l2.String()]
 	require.Equal(t, &indexEntry{0, 0, l2}, i)
 }
+
+func TestResize(t *testing.T) {
+	testCases := []struct {
+		name              string
+		startSize         int64
+		newCount          int64
+		expectedSeries    []int
+		notExpectedSeries []int
+		expectedMigrated  int
+	}{
+		{
+			name:              "Grow",
+			startSize:         100,
+			newCount:          200,
+			expectedSeries:    []int{99, 98, 1, 0},
+			notExpectedSeries: []int{100},
+			expectedMigrated:  100,
+		},
+		{
+			name:              "Shrink",
+			startSize:         100,
+			newCount:          50,
+			expectedSeries:    []int{99, 98, 50},
+			notExpectedSeries: []int{49, 1, 0},
+			expectedMigrated:  50,
+		},
+		{
+			name:              "Zero",
+			startSize:         100,
+			newCount:          0,
+			expectedSeries:    []int{},
+			notExpectedSeries: []int{},
+			expectedMigrated:  0,
+		},
+		{
+			name:              "Negative",
+			startSize:         100,
+			newCount:          -1,
+			expectedSeries:    []int{},
+			notExpectedSeries: []int{},
+			expectedMigrated:  0,
+		},
+		{
+			name:              "NegativeToNegative",
+			startSize:         -1,
+			newCount:          -2,
+			expectedSeries:    []int{},
+			notExpectedSeries: []int{},
+			expectedMigrated:  0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			exs, err := NewCircularExemplarStorage(tc.startSize, eMetrics)
+			require.NoError(t, err)
+			es := exs.(*CircularExemplarStorage)
+
+			for i := 0; int64(i) < tc.startSize; i++ {
+				err = es.AddExemplar(labels.FromStrings("service", strconv.Itoa(i)), exemplar.Exemplar{
+					Value: float64(i),
+					Ts:    int64(i)})
+				require.NoError(t, err)
+			}
+
+			resized := es.Resize(tc.newCount)
+			require.Equal(t, tc.expectedMigrated, resized)
+
+			q, err := es.Querier(context.TODO())
+			require.NoError(t, err)
+
+			matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "service", "")}
+
+			for _, expected := range tc.expectedSeries {
+				matchers[0].Value = strconv.Itoa(expected)
+				ex, err := q.Select(0, math.MaxInt64, matchers)
+				require.NoError(t, err)
+				require.NotEmpty(t, ex)
+			}
+
+			for _, notExpected := range tc.notExpectedSeries {
+				matchers[0].Value = strconv.Itoa(notExpected)
+				ex, err := q.Select(0, math.MaxInt64, matchers)
+				require.NoError(t, err)
+				require.Empty(t, ex)
+			}
+		})
+	}
+}
+
+func BenchmarkAddExemplar(t *testing.B) {
+	exs, err := NewCircularExemplarStorage(int64(t.N), eMetrics)
+	require.NoError(t, err)
+	es := exs.(*CircularExemplarStorage)
+
+	for i := 0; i < t.N; i++ {
+		l := labels.FromStrings("service", strconv.Itoa(i))
+
+		err = es.AddExemplar(l, exemplar.Exemplar{Value: float64(i), Ts: int64(i)})
+		require.NoError(t, err)
+	}
+}
+
+func BenchmarkResizeExemplars(b *testing.B) {
+	testCases := []struct {
+		name         string
+		startSize    int64
+		endSize      int64
+		numExemplars int
+	}{
+		{
+			name:         "grow",
+			startSize:    100000,
+			endSize:      200000,
+			numExemplars: 150000,
+		},
+		{
+			name:         "shrink",
+			startSize:    100000,
+			endSize:      50000,
+			numExemplars: 100000,
+		},
+		{
+			name:         "grow",
+			startSize:    1000000,
+			endSize:      2000000,
+			numExemplars: 1500000,
+		},
+		{
+			name:         "shrink",
+			startSize:    1000000,
+			endSize:      500000,
+			numExemplars: 1000000,
+		},
+	}
+
+	for _, tc := range testCases {
+		exs, err := NewCircularExemplarStorage(tc.startSize, eMetrics)
+		require.NoError(b, err)
+		es := exs.(*CircularExemplarStorage)
+
+		for i := 0; i < int(float64(tc.startSize)*float64(1.5)); i++ {
+			l := labels.FromStrings("service", strconv.Itoa(i))
+
+			err = es.AddExemplar(l, exemplar.Exemplar{Value: float64(i), Ts: int64(i)})
+			require.NoError(b, err)
+		}
+		saveIndex := es.index
+		saveExemplars := es.exemplars
+
+		b.Run(fmt.Sprintf("%s-%d-to-%d", tc.name, tc.startSize, tc.endSize), func(t *testing.B) {
+			es.index = saveIndex
+			es.exemplars = saveExemplars
+			b.ResetTimer()
+			es.Resize(tc.endSize)
+		})
+	}
+}