diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d540eaf2..071ed2ef68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## unreleased + * [ENHANCEMENT] Scraping: add warning if targets relabel to same labels. This is enabled under the feature-flag `warn-if-targets-relabelled-to-same-labels`. #9589 ## 3.2.1 / 2025-02-25 diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 4559d51837..40db744252 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -264,6 +264,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols logger.Info("Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) + case "warn-if-targets-relabelled-to-same-labels": + c.scrape.EnableWarnIfTargetsRelabelledToSameLabels = true + logger.Info("Enabled warning if targets relabelled to same labels") case "delayed-compaction": c.tsdb.EnableDelayedCompaction = true logger.Info("Experimental delayed compaction is enabled.") diff --git a/scrape/manager.go b/scrape/manager.go index 5ef5dccb99..51d4514b59 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -90,6 +90,9 @@ type Options struct { // Optional HTTP client options to use when scraping. HTTPClientOptions []config_util.HTTPClientOption + // Option to warn if targets relabelled to same labels + EnableWarnIfTargetsRelabelledToSameLabels bool + // private option for testability. skipOffsetting bool } @@ -205,6 +208,40 @@ func (m *Manager) reload() { } m.mtxScrape.Unlock() wg.Wait() + + if m.opts.EnableWarnIfTargetsRelabelledToSameLabels { + m.warnIfTargetsRelabelledToSameLabels() + } +} + +func (m *Manager) warnIfTargetsRelabelledToSameLabels() { + m.mtxScrape.Lock() + defer m.mtxScrape.Unlock() + + totalTargets := 0 + for _, scrapePool := range m.scrapePools { + totalTargets += len(scrapePool.activeTargets) + } + + activeTargets := make(map[string]*Target, totalTargets) + buf := [1024]byte{} + builder := labels.NewBuilder(labels.EmptyLabels()) + for _, scrapePool := range m.scrapePools { + for _, target := range scrapePool.activeTargets { + lStr := string(target.labels.Bytes(buf[:])) + t, ok := activeTargets[lStr] + if !ok { + activeTargets[lStr] = target + continue + } + m.logger.Warn( + "Found targets with same labels after relabelling", + "target_one", t.DiscoveredLabels(builder).Get(model.AddressLabel), + "target_two", target.DiscoveredLabels(builder).Get(model.AddressLabel), + "labels", target.labels.String(), + ) + } + } } // setOffsetSeed calculates a global offsetSeed per server relying on extra label set. diff --git a/scrape/manager_test.go b/scrape/manager_test.go index 96381fa736..309e9b7cad 100644 --- a/scrape/manager_test.go +++ b/scrape/manager_test.go @@ -622,6 +622,43 @@ func TestManagerTargetsUpdates(t *testing.T) { } } +func TestManagerDuplicateAfterRelabellingWarning(t *testing.T) { + var buf bytes.Buffer + writer := &buf + logger := promslog.New(&promslog.Config{Writer: writer}) + + opts := Options{EnableWarnIfTargetsRelabelledToSameLabels: true} + testRegistry := prometheus.NewRegistry() + m, err := NewManager(&opts, logger, nil, nil, testRegistry) + require.NoError(t, err) + + m.scrapePools = map[string]*scrapePool{} + sp := &scrapePool{ + activeTargets: map[uint64]*Target{}, + } + targetScrapeCfg := &config.ScrapeConfig{ + Scheme: "https", + MetricsPath: "/metrics", + JobName: "job", + ScrapeInterval: model.Duration(time.Second), + ScrapeTimeout: model.Duration(time.Second), + } + sp.activeTargets[uint64(0)] = &Target{ + scrapeConfig: targetScrapeCfg, + tLabels: map[model.LabelName]model.LabelValue{model.AddressLabel: "foo"}, + } + sp.activeTargets[uint64(1)] = &Target{ + scrapeConfig: targetScrapeCfg, + tLabels: map[model.LabelName]model.LabelValue{model.AddressLabel: "bar"}, + } + m.scrapePools["default"] = sp + + m.reload() + require.Contains(t, buf.String(), "Found targets with same labels after relabelling") + require.Contains(t, buf.String(), "foo") + require.Contains(t, buf.String(), "bar") +} + func TestSetOffsetSeed(t *testing.T) { getConfig := func(prometheus string) *config.Config { cfgText := `