mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-23 12:44:05 -08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Merge branch 'main' into sparsehistogram

Browse source
This commit is contained in:
beorn7 2021-11-17 19:57:31 +01:00
parent 742b504be8 0028c4f3ec
commit 5d4db805ac
324 changed files with 12047 additions and 4394 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
.circleci/config.yml
View file

@ -2,7 +2,7 @@
version: 2.1
orbs:
prometheus: prometheus/prometheus@0.11.0
prometheus: prometheus/prometheus@0.14.0
go: circleci/go@1.7.0
win: circleci/windows@2.3.0
@ -99,21 +99,13 @@ jobs:
steps:
- checkout
- run: go install ./cmd/promtool/.
- run:
command: go install -mod=readonly github.com/google/go-jsonnet/cmd/jsonnet github.com/google/go-jsonnet/cmd/jsonnetfmt github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: make clean
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: jb install
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: make
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: git diff --exit-code
working_directory: ~/project/documentation/prometheus-mixin
- run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest
- run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
- run: go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
- run: make -C documentation/prometheus-mixin clean
- run: make -C documentation/prometheus-mixin jb_install
- run: make -C documentation/prometheus-mixin
- run: git diff --exit-code
repo_sync:
executor: golang
@ -148,8 +140,19 @@ workflows:
only: /.*/
- prometheus/build:
name: build
parallelism: 3
promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
filters:
tags:
ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
branches:
ignore: /^(main|release-.*|.*build-all.*)$/
- prometheus/build:
name: build_all
parallelism: 12
filters:
branches:
only: /^(main|release-.*|.*build-all.*)$/
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
- prometheus/publish_main:
@ -157,7 +160,7 @@ workflows:
requires:
- test_go
- test_ui
- build
- build_all
filters:
branches:
only: main
@ -167,7 +170,7 @@ workflows:
requires:
- test_go
- test_ui
- build
- build_all
filters:
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/

18
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,18 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "npm"
directory: "/web/ui"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "weekly"

35
.github/lock.yml vendored
View file

@ -1,35 +0,0 @@
# Configuration for Lock Threads - https://github.com/dessant/lock-threads
# Number of days of inactivity before a closed issue or pull request is locked
daysUntilLock: 180
# Skip issues and pull requests created before a given timestamp. Timestamp must
# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
skipCreatedBefore: false
# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
exemptLabels: []
# Label to add before locking, such as `outdated`. Set to `false` to disable
lockLabel: false
# Comment to post before locking. Set to `false` to disable
lockComment: false
# Assign `resolved` as the reason for locking. Set to `false` to disable
setLockReason: false
# Limit to only `issues` or `pulls`
only: issues
# Optionally, specify configuration settings just for `issues` or `pulls`
# issues:
# exemptLabels:
# - help-wanted
# lockLabel: outdated
# pulls:
# daysUntilLock: 30
# Repository to extend settings from
# _extends: repo

2
.github/workflows/fuzzing.yml vendored
View file

@ -22,7 +22,7 @@ jobs:
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v2.2.4
if: failure() && steps.build.outcome == 'success'
with:
name: artifacts

2
.github/workflows/golangci-lint.yml vendored
View file

@ -7,6 +7,7 @@ on:
- "**.go"
- "scripts/errcheck_excludes.txt"
- ".github/workflows/golangci-lint.yml"
- ".golangci.yml"
pull_request:
paths:
- "go.sum"
@ -14,6 +15,7 @@ on:
- "**.go"
- "scripts/errcheck_excludes.txt"
- ".github/workflows/golangci-lint.yml"
- ".golangci.yml"
jobs:
golangci:

22
.github/workflows/lock.yml vendored Normal file
View file

@ -0,0 +1,22 @@
name: 'Lock Threads'
on:
schedule:
- cron: '13 23 * * *'
workflow_dispatch:
permissions:
issues: write
concurrency:
group: lock
jobs:
action:
runs-on: ubuntu-latest
steps:
- uses: dessant/lock-threads@v3
with:
process-only: 'issues'
issue-inactive-days: '180'
github-token: ${{ secrets.PROMBOT_LOCKTHREADS_TOKEN }}

2
.gitignore vendored
View file

@ -8,7 +8,9 @@
/promtool
benchmark.txt
/data
/data-agent
/cmd/prometheus/data
/cmd/prometheus/data-agent
/cmd/prometheus/debug
/benchout
/cmd/promtool/data

14
.golangci.yml
View file

@ -1,10 +1,18 @@
run:
deadline: 5m
skip-files:
# Skip autogenerated files.
- ^.*\.(pb|y)\.go$
output:
sort-results: true
linters:
enable:
- depguard
- golint
- gofumpt
- goimports
- revive
issues:
exclude-rules:
@ -25,3 +33,7 @@ linters-settings:
- github.com/go-kit/kit/log: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
errcheck:
exclude: scripts/errcheck_excludes.txt
goimports:
local-prefixes: github.com/prometheus/prometheus
gofumpt:
extra-rules: true

57
CHANGELOG.md
View file

@ -1,3 +1,58 @@
## 2.32.0-beta.0 / 2021-11-16
This beta release introduces the Prometheus Agent, a new mode of operation for
Prometheus optimized for remote-write only scenarios. In this mode, Prometheus
does not generate blocks on the local filesystem and is not queryable locally.
Enable with `--enable-feature=agent`.
Learn more about the Prometheus Agent in our [blog post](https://prometheus.io/blog/2021/11/16/agent/).
* [CHANGE] remote-write: Change default max retry time from 100ms to 5 seconds. #9634
* [FEATURE] Agent: New mode of operation optimized for remote-write only scenarios, without local storage. Enable with `--enable-feature=agent`. #8785
* [FEATURE] Promtool: Add `promtool check service-discovery` command. #8970
* [FEATURE] UI: Add search in metrics dropdown. #9629
* [FEATURE] Templates: Add parseDuration to template functions. #8817
* [ENHANCEMENT] Promtool: Improve test output. #8064
* [ENHANCEMENT] Promtool: Use kahan summation for better numerical stability. #9588
* [ENHANCEMENT] Remote-write: Reuse memory for marshalling. #9412
* [ENHANCEMENT] Scrape: Add `scrape_body_size_bytes` scrape metric behind the `--enable-feature=extra-scrape-metrics` flag. #9569
* [ENHANCEMENT] TSDB: Add windows arm64 support. #9703
* [ENHANCEMENT] TSDB: Optimize query by skipping unneeded sorting in TSDB. #9673
* [ENHANCEMENT] Templates: Support int and uint as datatypes for template formatting. #9680
* [ENHANCEMENT] UI: Prefer `rate` over `rad`, `delta` over `deg`, and `count` over `cos` in autocomplete. #9688
* [BUGFIX] TSDB: Add more size checks when writing individual sections in the index. #9710
* [BUGFIX] PromQL: Make `deriv()` return zero values for constant series. #9728
## 2.31.1 / 2021-11-05
* [BUGFIX] SD: Fix a panic when the experimental discovery manager receives
targets during a reload. #9656
## 2.31.0 / 2021-11-02
* [CHANGE] UI: Remove standard PromQL editor in favour of the codemirror-based editor. #9452
* [FEATURE] PromQL: Add trigonometric functions and `atan2` binary operator. #9239 #9248 #9515
* [FEATURE] Remote: Add support for exemplar in the remote write receiver endpoint. #9319 #9414
* [FEATURE] SD: Add PuppetDB service discovery. #8883
* [FEATURE] SD: Add Uyuni service discovery. #8190
* [FEATURE] Web: Add support for security-related HTTP headers. #9546
* [ENHANCEMENT] Azure SD: Add `proxy_url`, `follow_redirects`, `tls_config`. #9267
* [ENHANCEMENT] Backfill: Add `--max-block-duration` in `promtool create-blocks-from rules`. #9511
* [ENHANCEMENT] Config: Print human-readable sizes with unit instead of raw numbers. #9361
* [ENHANCEMENT] HTTP: Re-enable HTTP/2. #9398
* [ENHANCEMENT] Kubernetes SD: Warn user if number of endpoints exceeds limit. #9467
* [ENHANCEMENT] OAuth2: Add TLS configuration to token requests. #9550
* [ENHANCEMENT] PromQL: Several optimizations. #9365 #9360 #9362 #9552
* [ENHANCEMENT] PromQL: Make aggregations deterministic in instant queries. #9459
* [ENHANCEMENT] Rules: Add the ability to limit number of alerts or series. #9260 #9541
* [ENHANCEMENT] SD: Experimental discovery manager to avoid restarts upon reload. Disabled by default, enable with flag `--enable-feature=new-service-discovery-manager`. #9349 #9537
* [ENHANCEMENT] UI: Debounce timerange setting changes. #9359
* [BUGFIX] Backfill: Apply rule labels after query labels. #9421
* [BUGFIX] Scrape: Resolve conflicts between multiple exported label prefixes. #9479 #9518
* [BUGFIX] Scrape: Restart scrape loops when `__scrape_interval__` is changed. #9551
* [BUGFIX] TSDB: Fix memory leak in samples deletion. #9151
* [BUGFIX] UI: Use consistent margin-bottom for all alert kinds. #9318
## 2.30.3 / 2021-10-05
* [BUGFIX] TSDB: Fix panic on failed snapshot replay. #9438
@ -191,7 +246,7 @@ Alertmanager API v2 was released in Alertmanager v0.16.0 (released in January
* [ENHANCEMENT] Mixins: Scope grafana configuration. #8332
* [ENHANCEMENT] Kubernetes SD: Add endpoint labels metadata. #8273
* [ENHANCEMENT] UI: Expose total number of label pairs in head in TSDB stats page. #8343
* [ENHANCEMENT] TSDB: Reload blocks every minute, to detect new blocks and enforce retention more often. #8343
* [ENHANCEMENT] TSDB: Reload blocks every minute, to detect new blocks and enforce retention more often. #8340
* [BUGFIX] API: Fix global URL when external address has no port. #8359
* [BUGFIX] Backfill: Fix error message handling. #8432
* [BUGFIX] Backfill: Fix "add sample: out of bounds" error when series span an entire block. #8476

1
MAINTAINERS.md
View file

@ -9,6 +9,7 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
* `storage`
* `remote`: Chris Marchbanks (<csmarchbanks@gmail.com> / @csmarchbanks), Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
* `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka)
* `agent`: Robert Fratto (<robert.fratto@grafana.com> / @rfratto)
* `web`
* `ui`: Julius Volz (<julius.volz@gmail.com> / @juliusv)
* `module`: Augustin Husson (<husson.augustin@gmail.com> @nexucis)

4
Makefile
View file

@ -55,8 +55,8 @@ assets: ui-install ui-build
# Un-setting GOOS and GOARCH here because the generated Go code is always the same,
# but the cached object code is incompatible between architectures and OSes (which
# breaks cross-building for different combinations on CI in the same container).
cd web/ui && GO111MODULE=$(GO111MODULE) GOOS= GOARCH= $(GO) generate -x -v $(GOOPTS)
@$(GOFMT) -w ./web/ui
cd $(UI_PATH) && GO111MODULE=$(GO111MODULE) GOOS= GOARCH= $(GO) generate -x -v $(GOOPTS)
@$(GOFMT) -w ./$(UI_PATH)
.PHONY: test
# If we only want to only test go code we have to change the test target

4
Makefile.common
View file

@ -78,7 +78,7 @@ ifneq ($(shell which gotestsum),)
endif
endif
PROMU_VERSION ?= 0.12.0
PROMU_VERSION ?= 0.13.0
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
GOLANGCI_LINT :=
@ -160,7 +160,7 @@ endif
update-go-deps:
@echo ">> updating Go dependencies"
@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
$(GO) get $$m; \
$(GO) get -d $$m; \
done
GO111MODULE=$(GO111MODULE) $(GO) mod tidy
ifneq (,$(wildcard vendor))

4
README.md
View file

@ -107,11 +107,11 @@ You can build a docker image locally with the following commands:
## React UI Development
For more information on building, running, and developing on the new React-based UI, see the React app's [README.md](web/ui/react-app/README.md).
For more information on building, running, and developing on the new React-based UI, see the React app's [README.md](web/ui/README.md).
## More information
* The source code is periodically indexed: [Prometheus Core](https://pkg.go.dev/github.com/prometheus/prometheus).
* The source code is periodically indexed, but due to an issue with versioning, the "latest" docs shown on Godoc are outdated. Instead, you can use [the docs for v2.31.1](https://pkg.go.dev/github.com/prometheus/prometheus@v1.8.2-0.20211105201321-411021ada9ab).
* You will find a CircleCI configuration in [`.circleci/config.yml`](.circleci/config.yml).
* See the [Community page](https://prometheus.io/community) for how to reach the Prometheus developers and users on various communication channels.

2
VERSION
View file

@ -1 +1 @@
2.30.3
2.32.0-beta.0

384
cmd/prometheus/main.go
View file

@ -58,24 +58,29 @@ import (
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
"github.com/prometheus/prometheus/discovery/legacymanager"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/pkg/exemplar"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/logging"
"github.com/prometheus/prometheus/pkg/relabel"
prom_runtime "github.com/prometheus/prometheus/pkg/runtime"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/scrape"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/agent"
"github.com/prometheus/prometheus/util/logging"
prom_runtime "github.com/prometheus/prometheus/util/runtime"
"github.com/prometheus/prometheus/util/strutil"
"github.com/prometheus/prometheus/web"
)
var (
appName = "prometheus"
configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "prometheus_config_last_reload_successful",
Help: "Whether the last configuration reload attempt was successful.",
@ -87,10 +92,13 @@ var (
defaultRetentionString = "15d"
defaultRetentionDuration model.Duration
agentMode bool
agentOnlyFlags, serverOnlyFlags []string
)
func init() {
prometheus.MustRegister(version.NewCollector("prometheus"))
prometheus.MustRegister(version.NewCollector(strings.ReplaceAll(appName, "-", "_")))
var err error
defaultRetentionDuration, err = model.ParseDuration(defaultRetentionString)
@ -99,10 +107,31 @@ func init() {
}
}
// serverOnlyFlag creates server-only kingpin flag.
func serverOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause {
return app.Flag(name, fmt.Sprintf("%s Use with server mode only.", help)).
PreAction(func(parseContext *kingpin.ParseContext) error {
// This will be invoked only if flag is actually provided by user.
serverOnlyFlags = append(serverOnlyFlags, "--"+name)
return nil
})
}
// agentOnlyFlag creates agent-only kingpin flag.
func agentOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause {
return app.Flag(name, fmt.Sprintf("%s Use with agent mode only.", help)).
PreAction(func(parseContext *kingpin.ParseContext) error {
// This will be invoked only if flag is actually provided by user.
agentOnlyFlags = append(agentOnlyFlags, "--"+name)
return nil
})
}
type flagConfig struct {
configFile string
localStoragePath string
agentStoragePath string
serverStoragePath string
notifier notifier.Options
forGracePeriod model.Duration
outageTolerance model.Duration
@ -110,6 +139,7 @@ type flagConfig struct {
web web.Options
scrape scrape.Options
tsdb tsdbOptions
agent agentOptions
lookbackDelta model.Duration
webTimeout model.Duration
queryTimeout model.Duration
@ -123,6 +153,7 @@ type flagConfig struct {
enablePromQLAtModifier bool
enablePromQLNegativeOffset bool
enableExpandExternalLabels bool
enableNewSDManager bool
prometheusURL string
corsRegexString string
@ -157,6 +188,12 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
case "extra-scrape-metrics":
c.scrape.ExtraMetrics = true
level.Info(logger).Log("msg", "Experimental additional scrape metrics")
case "new-service-discovery-manager":
c.enableNewSDManager = true
level.Info(logger).Log("msg", "Experimental service discovery manager")
case "agent":
agentMode = true
level.Info(logger).Log("msg", "Experimental agent mode enabled.")
case "":
continue
default:
@ -191,7 +228,7 @@ func main() {
a := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus monitoring server").UsageWriter(os.Stdout)
a.Version(version.Print("prometheus"))
a.Version(version.Print(appName))
a.HelpFlag.Short('h')
@ -239,61 +276,86 @@ func main() {
a.Flag("web.cors.origin", `Regex for CORS origin. It is fully anchored. Example: 'https?://(domain1|domain2)\.com'`).
Default(".*").StringVar(&cfg.corsRegexString)
a.Flag("storage.tsdb.path", "Base path for metrics storage.").
Default("data/").StringVar(&cfg.localStoragePath)
serverOnlyFlag(a, "storage.tsdb.path", "Base path for metrics storage.").
Default("data/").StringVar(&cfg.serverStoragePath)
a.Flag("storage.tsdb.min-block-duration", "Minimum duration of a data block before being persisted. For use in testing.").
serverOnlyFlag(a, "storage.tsdb.min-block-duration", "Minimum duration of a data block before being persisted. For use in testing.").
Hidden().Default("2h").SetValue(&cfg.tsdb.MinBlockDuration)
a.Flag("storage.tsdb.max-block-duration",
serverOnlyFlag(a, "storage.tsdb.max-block-duration",
"Maximum duration compacted blocks may span. For use in testing. (Defaults to 10% of the retention period.)").
Hidden().PlaceHolder("<duration>").SetValue(&cfg.tsdb.MaxBlockDuration)
a.Flag("storage.tsdb.max-block-chunk-segment-size",
serverOnlyFlag(a, "storage.tsdb.max-block-chunk-segment-size",
"The maximum size for a single chunk segment in a block. Example: 512MB").
Hidden().PlaceHolder("<bytes>").BytesVar(&cfg.tsdb.MaxBlockChunkSegmentSize)
a.Flag("storage.tsdb.wal-segment-size",
serverOnlyFlag(a, "storage.tsdb.wal-segment-size",
"Size at which to split the tsdb WAL segment files. Example: 100MB").
Hidden().PlaceHolder("<bytes>").BytesVar(&cfg.tsdb.WALSegmentSize)
a.Flag("storage.tsdb.retention", "[DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use \"storage.tsdb.retention.time\" instead.").
serverOnlyFlag(a, "storage.tsdb.retention", "[DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use \"storage.tsdb.retention.time\" instead.").
SetValue(&oldFlagRetentionDuration)
a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
serverOnlyFlag(a, "storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
SetValue(&newFlagRetentionDuration)
a.Flag("storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\".").
serverOnlyFlag(a, "storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\".").
BytesVar(&cfg.tsdb.MaxBytes)
a.Flag("storage.tsdb.no-lockfile", "Do not create lockfile in data directory.").
serverOnlyFlag(a, "storage.tsdb.no-lockfile", "Do not create lockfile in data directory.").
Default("false").BoolVar(&cfg.tsdb.NoLockfile)
a.Flag("storage.tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").
serverOnlyFlag(a, "storage.tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").
Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks)
a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL.").
serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL.").
Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression)
agentOnlyFlag(a, "storage.agent.path", "Base path for metrics storage.").
Default("data-agent/").StringVar(&cfg.agentStoragePath)
agentOnlyFlag(a, "storage.agent.wal-segment-size",
"Size at which to split WAL segment files. Example: 100MB").
Hidden().PlaceHolder("<bytes>").BytesVar(&cfg.agent.WALSegmentSize)
agentOnlyFlag(a, "storage.agent.wal-compression", "Compress the agent WAL.").
Default("true").BoolVar(&cfg.agent.WALCompression)
agentOnlyFlag(a, "storage.agent.wal-truncate-frequency",
"The frequency at which to truncate the WAL and remove old data.").
Hidden().PlaceHolder("<duration>").SetValue(&cfg.agent.TruncateFrequency)
agentOnlyFlag(a, "storage.agent.retention.min-time",
"Minimum age samples may be before being considered for deletion when the WAL is truncated").
SetValue(&cfg.agent.MinWALTime)
agentOnlyFlag(a, "storage.agent.retention.max-time",
"Maximum age samples may be before being forcibly deleted when the WAL is truncated").
SetValue(&cfg.agent.MaxWALTime)
agentOnlyFlag(a, "storage.agent.no-lockfile", "Do not create lockfile in data directory.").
Default("false").BoolVar(&cfg.agent.NoLockfile)
a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload.").
Default("1m").PlaceHolder("<duration>").SetValue(&cfg.RemoteFlushDeadline)
a.Flag("storage.remote.read-sample-limit", "Maximum overall number of samples to return via the remote read interface, in a single query. 0 means no limit. This limit is ignored for streamed response types.").
serverOnlyFlag(a, "storage.remote.read-sample-limit", "Maximum overall number of samples to return via the remote read interface, in a single query. 0 means no limit. This limit is ignored for streamed response types.").
Default("5e7").IntVar(&cfg.web.RemoteReadSampleLimit)
a.Flag("storage.remote.read-concurrent-limit", "Maximum number of concurrent remote read calls. 0 means no limit.").
serverOnlyFlag(a, "storage.remote.read-concurrent-limit", "Maximum number of concurrent remote read calls. 0 means no limit.").
Default("10").IntVar(&cfg.web.RemoteReadConcurrencyLimit)
a.Flag("storage.remote.read-max-bytes-in-frame", "Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default.").
serverOnlyFlag(a, "storage.remote.read-max-bytes-in-frame", "Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default.").
Default("1048576").IntVar(&cfg.web.RemoteReadBytesInFrame)
a.Flag("rules.alert.for-outage-tolerance", "Max time to tolerate prometheus outage for restoring \"for\" state of alert.").
serverOnlyFlag(a, "rules.alert.for-outage-tolerance", "Max time to tolerate prometheus outage for restoring \"for\" state of alert.").
Default("1h").SetValue(&cfg.outageTolerance)
a.Flag("rules.alert.for-grace-period", "Minimum duration between alert and restored \"for\" state. This is maintained only for alerts with configured \"for\" time greater than grace period.").
serverOnlyFlag(a, "rules.alert.for-grace-period", "Minimum duration between alert and restored \"for\" state. This is maintained only for alerts with configured \"for\" time greater than grace period.").
Default("10m").SetValue(&cfg.forGracePeriod)
a.Flag("rules.alert.resend-delay", "Minimum amount of time to wait before resending an alert to Alertmanager.").
serverOnlyFlag(a, "rules.alert.resend-delay", "Minimum amount of time to wait before resending an alert to Alertmanager.").
Default("1m").SetValue(&cfg.resendDelay)
a.Flag("scrape.adjust-timestamps", "Adjust scrape timestamps by up to `scrape.timestamp-tolerance` to align them to the intended schedule. See https://github.com/prometheus/prometheus/issues/7846 for more context. Experimental. This flag will be removed in a future release.").
@ -302,25 +364,25 @@ func main() {
a.Flag("scrape.timestamp-tolerance", "Timestamp tolerance. See https://github.com/prometheus/prometheus/issues/7846 for more context. Experimental. This flag will be removed in a future release.").
Hidden().Default("2ms").DurationVar(&scrape.ScrapeTimestampTolerance)
a.Flag("alertmanager.notification-queue-capacity", "The capacity of the queue for pending Alertmanager notifications.").
serverOnlyFlag(a, "alertmanager.notification-queue-capacity", "The capacity of the queue for pending Alertmanager notifications.").
Default("10000").IntVar(&cfg.notifier.QueueCapacity)
// TODO: Remove in Prometheus 3.0.
alertmanagerTimeout := a.Flag("alertmanager.timeout", "[DEPRECATED] This flag has no effect.").Hidden().String()
a.Flag("query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations and federation.").
serverOnlyFlag(a, "query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations and federation.").
Default("5m").SetValue(&cfg.lookbackDelta)
a.Flag("query.timeout", "Maximum time a query may take before being aborted.").
serverOnlyFlag(a, "query.timeout", "Maximum time a query may take before being aborted.").
Default("2m").SetValue(&cfg.queryTimeout)
a.Flag("query.max-concurrency", "Maximum number of queries executed concurrently.").
serverOnlyFlag(a, "query.max-concurrency", "Maximum number of queries executed concurrently.").
Default("20").IntVar(&cfg.queryConcurrency)
a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return.").
serverOnlyFlag(a, "query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return.").
Default("50000000").IntVar(&cfg.queryMaxSamples)
a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, remote-write-receiver, extra-scrape-metrics. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, remote-write-receiver, extra-scrape-metrics, new-service-discovery-manager. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
Default("").StringsVar(&cfg.featureList)
promlogflag.AddFlags(a, &cfg.promlogConfig)
@ -339,6 +401,21 @@ func main() {
os.Exit(1)
}
if agentMode && len(serverOnlyFlags) > 0 {
fmt.Fprintf(os.Stderr, "The following flag(s) can not be used in agent mode: %q", serverOnlyFlags)
os.Exit(3)
}
if !agentMode && len(agentOnlyFlags) > 0 {
fmt.Fprintf(os.Stderr, "The following flag(s) can only be used in agent mode: %q", agentOnlyFlags)
os.Exit(3)
}
localStoragePath := cfg.serverStoragePath
if agentMode {
localStoragePath = cfg.agentStoragePath
}
cfg.web.ExternalURL, err = computeExternalURL(cfg.prometheusURL, cfg.web.ListenAddress)
if err != nil {
fmt.Fprintln(os.Stderr, errors.Wrapf(err, "parse external URL %q", cfg.prometheusURL))
@ -357,7 +434,7 @@ func main() {
// Throw error for invalid config before starting other components.
var cfgFile *config.Config
if cfgFile, err = config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil {
if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, false, log.NewNopLogger()); err != nil {
level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "err", err)
os.Exit(2)
}
@ -385,7 +462,8 @@ func main() {
// RoutePrefix must always be at least '/'.
cfg.web.RoutePrefix = "/" + strings.Trim(cfg.web.RoutePrefix, "/")
{ // Time retention settings.
if !agentMode {
// Time retention settings.
if oldFlagRetentionDuration != 0 {
level.Warn(logger).Log("deprecation_notice", "'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead.")
cfg.tsdb.RetentionDuration = oldFlagRetentionDuration
@ -410,9 +488,8 @@ func main() {
cfg.tsdb.RetentionDuration = y
level.Warn(logger).Log("msg", "Time retention value is too high. Limiting to: "+y.String())
}
}
{ // Max block size settings.
// Max block size settings.
if cfg.tsdb.MaxBlockDuration == 0 {
maxBlockDuration, err := model.ParseDuration("31d")
if err != nil {
@ -449,7 +526,7 @@ func main() {
var (
localStorage = &readyStorage{stats: tsdb.NewDBStats()}
scraper = &readyScrapeManager{}
remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, cfg.localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper)
remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper)
fanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage)
)
@ -460,19 +537,35 @@ func main() {
notifierManager = notifier.NewManager(&cfg.notifier, log.With(logger, "component", "notifier"))
ctxScrape, cancelScrape = context.WithCancel(context.Background())
discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
ctxNotify, cancelNotify = context.WithCancel(context.Background())
discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
discoveryManagerScrape discoveryManager
discoveryManagerNotify discoveryManager
)
if cfg.enableNewSDManager {
discovery.RegisterMetrics()
discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
} else {
legacymanager.RegisterMetrics()
discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), legacymanager.Name("scrape"))
discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), legacymanager.Name("notify"))
}
var (
scrapeManager = scrape.NewManager(&cfg.scrape, log.With(logger, "component", "scrape manager"), fanoutStorage)
opts = promql.EngineOpts{
queryEngine *promql.Engine
ruleManager *rules.Manager
)
if !agentMode {
opts := promql.EngineOpts{
Logger: log.With(logger, "component", "query engine"),
Reg: prometheus.DefaultRegisterer,
MaxSamples: cfg.queryMaxSamples,
Timeout: time.Duration(cfg.queryTimeout),
ActiveQueryTracker: promql.NewActiveQueryTracker(cfg.localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")),
ActiveQueryTracker: promql.NewActiveQueryTracker(localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")),
LookbackDelta: time.Duration(cfg.lookbackDelta),
NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
EnableAtModifier: cfg.enablePromQLAtModifier,
@ -494,14 +587,14 @@ func main() {
ForGracePeriod: time.Duration(cfg.forGracePeriod),
ResendDelay: time.Duration(cfg.resendDelay),
})
)
}
scraper.Set(scrapeManager)
cfg.web.Context = ctxWeb
cfg.web.TSDBRetentionDuration = cfg.tsdb.RetentionDuration
cfg.web.TSDBMaxBytes = cfg.tsdb.MaxBytes
cfg.web.TSDBDir = cfg.localStoragePath
cfg.web.TSDBDir = localStoragePath
cfg.web.LocalStorage = localStorage
cfg.web.Storage = fanoutStorage
cfg.web.ExemplarStorage = localStorage
@ -510,6 +603,7 @@ func main() {
cfg.web.RuleManager = ruleManager
cfg.web.Notifier = notifierManager
cfg.web.LookbackDelta = time.Duration(cfg.lookbackDelta)
cfg.web.IsAgent = agentMode
cfg.web.Version = &web.PrometheusVersion{
Version: version.Version,
@ -541,7 +635,7 @@ func main() {
)
// This is passed to ruleManager.Update().
var externalURL = cfg.web.ExternalURL.String()
externalURL := cfg.web.ExternalURL.String()
reloaders := []reloader{
{
@ -556,6 +650,11 @@ func main() {
}, {
name: "query_engine",
reloader: func(cfg *config.Config) error {
if agentMode {
// No-op in Agent mode.
return nil
}
if cfg.GlobalConfig.QueryLogFile == "" {
queryEngine.SetQueryLogger(nil)
return nil
@ -597,6 +696,11 @@ func main() {
}, {
name: "rules",
reloader: func(cfg *config.Config) error {
if agentMode {
// No-op in Agent mode
return nil
}
// Get all rule files matching the configuration paths.
var files []string
for _, pat := range cfg.RuleFiles {
@ -763,7 +867,6 @@ func main() {
return nil
}
}
},
func(err error) {
// Wait for any in-progress reloads to complete to avoid
@ -801,7 +904,7 @@ func main() {
},
)
}
{
if !agentMode {
// Rule manager.
g.Add(
func() error {
@ -813,8 +916,7 @@ func main() {
ruleManager.Stop()
},
)
}
{
// TSDB.
opts := cfg.tsdb.ToTSDBOptions()
cancel := make(chan struct{})
@ -832,18 +934,12 @@ func main() {
}
}
db, err := openDBWithMetrics(
cfg.localStoragePath,
logger,
prometheus.DefaultRegisterer,
&opts,
localStorage.getStats(),
)
db, err := openDBWithMetrics(localStoragePath, logger, prometheus.DefaultRegisterer, &opts, localStorage.getStats())
if err != nil {
return errors.Wrapf(err, "opening storage failed")
}
switch fsType := prom_runtime.Statfs(cfg.localStoragePath); fsType {
switch fsType := prom_runtime.Statfs(localStoragePath); fsType {
case "NFS_SUPER_MAGIC":
level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.")
default:
@ -876,6 +972,59 @@ func main() {
},
)
}
if agentMode {
// WAL storage.
opts := cfg.agent.ToAgentOptions()
cancel := make(chan struct{})
g.Add(
func() error {
level.Info(logger).Log("msg", "Starting WAL storage ...")
if cfg.agent.WALSegmentSize != 0 {
if cfg.agent.WALSegmentSize < 10*1024*1024 || cfg.agent.WALSegmentSize > 256*1024*1024 {
return errors.New("flag 'storage.agent.wal-segment-size' must be set between 10MB and 256MB")
}
}
db, err := agent.Open(
logger,
prometheus.DefaultRegisterer,
remoteStorage,
localStoragePath,
&opts,
)
if err != nil {
return errors.Wrap(err, "opening storage failed")
}
switch fsType := prom_runtime.Statfs(localStoragePath); fsType {
case "NFS_SUPER_MAGIC":
level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.")
default:
level.Info(logger).Log("fs_type", fsType)
}
level.Info(logger).Log("msg", "Agent WAL storage started")
level.Debug(logger).Log("msg", "Agent WAL storage options",
"WALSegmentSize", cfg.agent.WALSegmentSize,
"WALCompression", cfg.agent.WALCompression,
"StripeSize", cfg.agent.StripeSize,
"TruncateFrequency", cfg.agent.TruncateFrequency,
"MinWALTime", cfg.agent.MinWALTime,
"MaxWALTime", cfg.agent.MaxWALTime,
)
localStorage.Set(db, 0)
close(dbOpen)
<-cancel
return nil
},
func(e error) {
if err := fanoutStorage.Close(); err != nil {
level.Error(logger).Log("msg", "Error stopping storage", "err", err)
}
close(cancel)
},
)
}
{
// Web handler.
g.Add(
@ -961,6 +1110,7 @@ type safePromQLNoStepSubqueryInterval struct {
func durationToInt64Millis(d time.Duration) int64 {
return int64(d / time.Millisecond)
}
func (i *safePromQLNoStepSubqueryInterval) Set(ev model.Duration) {
i.value.Store(durationToInt64Millis(time.Duration(ev)))
}
@ -974,7 +1124,7 @@ type reloader struct {
reloader func(*config.Config) error
}
func reloadConfig(filename string, expandExternalLabels bool, enableExemplarStorage bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) {
func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) {
start := time.Now()
timings := []interface{}{}
level.Info(logger).Log("msg", "Loading configuration file", "filename", filename)
@ -988,7 +1138,7 @@ func reloadConfig(filename string, expandExternalLabels bool, enableExemplarStor
}
}()
conf, err := config.LoadFile(filename, expandExternalLabels, logger)
conf, err := config.LoadFile(filename, agentMode, expandExternalLabels, logger)
if err != nil {
return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename)
}
@ -1099,18 +1249,21 @@ func sendAlerts(s sender, externalURL string) rules.NotifyFunc {
// storage at a later point in time.
type readyStorage struct {
mtx sync.RWMutex
db *tsdb.DB
db storage.Storage
startTimeMargin int64
stats *tsdb.DBStats
}
func (s *readyStorage) ApplyConfig(conf *config.Config) error {
db := s.get()
return db.ApplyConfig(conf)
if db, ok := db.(*tsdb.DB); ok {
return db.ApplyConfig(conf)
}
return nil
}
// Set the storage.
func (s *readyStorage) Set(db *tsdb.DB, startTimeMargin int64) {
func (s *readyStorage) Set(db storage.Storage, startTimeMargin int64) {
s.mtx.Lock()
defer s.mtx.Unlock()
@ -1118,7 +1271,7 @@ func (s *readyStorage) Set(db *tsdb.DB, startTimeMargin int64) {
s.startTimeMargin = startTimeMargin
}
func (s *readyStorage) get() *tsdb.DB {
func (s *readyStorage) get() storage.Storage {
s.mtx.RLock()
x := s.db
s.mtx.RUnlock()
@ -1135,15 +1288,21 @@ func (s *readyStorage) getStats() *tsdb.DBStats {
// StartTime implements the Storage interface.
func (s *readyStorage) StartTime() (int64, error) {
if x := s.get(); x != nil {
var startTime int64
if len(x.Blocks()) > 0 {
startTime = x.Blocks()[0].Meta().MinTime
} else {
startTime = time.Now().Unix() * 1000
switch db := x.(type) {
case *tsdb.DB:
var startTime int64
if len(db.Blocks()) > 0 {
startTime = db.Blocks()[0].Meta().MinTime
} else {
startTime = time.Now().Unix() * 1000
}
// Add a safety margin as it may take a few minutes for everything to spin up.
return startTime + s.startTimeMargin, nil
case *agent.DB:
return db.StartTime()
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
// Add a safety margin as it may take a few minutes for everything to spin up.
return startTime + s.startTimeMargin, nil
}
return math.MaxInt64, tsdb.ErrNotReady
@ -1167,7 +1326,14 @@ func (s *readyStorage) ChunkQuerier(ctx context.Context, mint, maxt int64) (stor
func (s *readyStorage) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) {
if x := s.get(); x != nil {
return x.ExemplarQuerier(ctx)
switch db := x.(type) {
case *tsdb.DB:
return db.ExemplarQuerier(ctx)
case *agent.DB:
return nil, agent.ErrUnsupported
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
}
return nil, tsdb.ErrNotReady
}
@ -1182,15 +1348,15 @@ func (s *readyStorage) Appender(ctx context.Context) storage.Appender {
type notReadyAppender struct{}
func (n notReadyAppender) Append(ref uint64, l labels.Labels, t int64, v float64) (uint64, error) {
func (n notReadyAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
return 0, tsdb.ErrNotReady
}
func (n notReadyAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) {
func (n notReadyAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
return 0, tsdb.ErrNotReady
}
func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h *histogram.Histogram) (uint64, error) {
func (n notReadyAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram) (storage.SeriesRef, error) {
return 0, tsdb.ErrNotReady
}
@ -1209,7 +1375,14 @@ func (s *readyStorage) Close() error {
// CleanTombstones implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) CleanTombstones() error {
if x := s.get(); x != nil {
return x.CleanTombstones()
switch db := x.(type) {
case *tsdb.DB:
return db.CleanTombstones()
case *agent.DB:
return agent.ErrUnsupported
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
}
return tsdb.ErrNotReady
}
@ -1217,7 +1390,14 @@ func (s *readyStorage) CleanTombstones() error {
// Delete implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
if x := s.get(); x != nil {
return x.Delete(mint, maxt, ms...)
switch db := x.(type) {
case *tsdb.DB:
return db.Delete(mint, maxt, ms...)
case *agent.DB:
return agent.ErrUnsupported
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
}
return tsdb.ErrNotReady
}
@ -1225,7 +1405,14 @@ func (s *readyStorage) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
// Snapshot implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) Snapshot(dir string, withHead bool) error {
if x := s.get(); x != nil {
return x.Snapshot(dir, withHead)
switch db := x.(type) {
case *tsdb.DB:
return db.Snapshot(dir, withHead)
case *agent.DB:
return agent.ErrUnsupported
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
}
return tsdb.ErrNotReady
}
@ -1233,7 +1420,14 @@ func (s *readyStorage) Snapshot(dir string, withHead bool) error {
// Stats implements the api_v1.TSDBAdminStats interface.
func (s *readyStorage) Stats(statsByLabelName string) (*tsdb.Stats, error) {
if x := s.get(); x != nil {
return x.Head().Stats(statsByLabelName), nil
switch db := x.(type) {
case *tsdb.DB:
return db.Head().Stats(statsByLabelName), nil
case *agent.DB:
return nil, agent.ErrUnsupported
default:
panic(fmt.Sprintf("unknown storage type %T", db))
}
}
return nil, tsdb.ErrNotReady
}
@ -1311,6 +1505,29 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
}
}
// agentOptions is a version of agent.Options with defined units. This is required
// as agent.Option fields are unit agnostic (time).
type agentOptions struct {
WALSegmentSize units.Base2Bytes
WALCompression bool
StripeSize int
TruncateFrequency model.Duration
MinWALTime, MaxWALTime model.Duration
NoLockfile bool
}
func (opts agentOptions) ToAgentOptions() agent.Options {
return agent.Options{
WALSegmentSize: int(opts.WALSegmentSize),
WALCompression: opts.WALCompression,
StripeSize: opts.StripeSize,
TruncateFrequency: time.Duration(opts.TruncateFrequency),
MinWALTime: durationToInt64Millis(time.Duration(opts.MinWALTime)),
MaxWALTime: durationToInt64Millis(time.Duration(opts.MaxWALTime)),
NoLockfile: opts.NoLockfile,
}
}
func initTracing(logger log.Logger) (io.Closer, error) {
// Set tracing configuration defaults.
cfg := &jcfg.Configuration{
@ -1351,3 +1568,12 @@ func (l jaegerLogger) Infof(msg string, args ...interface{}) {
keyvals := []interface{}{"msg", fmt.Sprintf(msg, args...)}
level.Info(l.logger).Log(keyvals...)
}
// discoveryManager interfaces the discovery manager. This is used to keep using
// the manager that restarts SD's on reload for a few releases until we feel
// the new manager can be enabled for all users.
type discoveryManager interface {
ApplyConfig(cfg map[string]discovery.Configs) error
Run() error
SyncCh() <-chan map[string][]*targetgroup.Group
}

148
cmd/prometheus/main_test.go
View file

@ -14,6 +14,7 @@
package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
@ -21,6 +22,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
@ -30,14 +32,16 @@ import (
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/rules"
)
var promPath = os.Args[0]
var promConfig = filepath.Join("..", "..", "documentation", "examples", "prometheus.yml")
var promData = filepath.Join(os.TempDir(), "data")
var (
promPath = os.Args[0]
promConfig = filepath.Join("..", "..", "documentation", "examples", "prometheus.yml")
agentConfig = filepath.Join("..", "..", "documentation", "examples", "prometheus-agent.yml")
)
func TestMain(m *testing.M) {
for i, arg := range os.Args {
@ -52,7 +56,6 @@ func TestMain(m *testing.M) {
os.Setenv("no_proxy", "localhost,127.0.0.1,0.0.0.0,:")
exitCode := m.Run()
os.RemoveAll(promData)
os.Exit(exitCode)
}
@ -202,7 +205,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
}
for size, expectedExitStatus := range map[string]int{"9MB": 1, "257MB": 1, "10": 2, "1GB": 1, "12MB": 0} {
prom := exec.Command(promPath, "-test.main", "--storage.tsdb.wal-segment-size="+size, "--config.file="+promConfig)
prom := exec.Command(promPath, "-test.main", "--storage.tsdb.wal-segment-size="+size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data"))
// Log stderr in case of failure.
stderr, err := prom.StderrPipe()
@ -223,6 +226,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
t.Errorf("prometheus should be still running: %v", err)
case <-time.After(5 * time.Second):
prom.Process.Kill()
<-done
}
continue
}
@ -239,12 +243,14 @@ func TestWALSegmentSizeBounds(t *testing.T) {
}
func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("skipping test in short mode.")
}
for size, expectedExitStatus := range map[string]int{"512KB": 1, "1MB": 0} {
prom := exec.Command(promPath, "-test.main", "--storage.tsdb.max-block-chunk-segment-size="+size, "--config.file="+promConfig)
prom := exec.Command(promPath, "-test.main", "--storage.tsdb.max-block-chunk-segment-size="+size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data"))
// Log stderr in case of failure.
stderr, err := prom.StderrPipe()
@ -265,6 +271,7 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
t.Errorf("prometheus should be still running: %v", err)
case <-time.After(5 * time.Second):
prom.Process.Kill()
<-done
}
continue
}
@ -347,3 +354,130 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
}
return res
}
func TestAgentSuccessfulStartup(t *testing.T) {
prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--config.file="+agentConfig)
require.NoError(t, prom.Start())
actualExitStatus := 0
done := make(chan error, 1)
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Logf("prometheus agent should be still running: %v", err)
actualExitStatus = prom.ProcessState.ExitCode()
case <-time.After(5 * time.Second):
prom.Process.Kill()
}
require.Equal(t, 0, actualExitStatus)
}
func TestAgentFailedStartupWithServerFlag(t *testing.T) {
prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--storage.tsdb.path=.", "--config.file="+promConfig)
output := bytes.Buffer{}
prom.Stderr = &output
require.NoError(t, prom.Start())
actualExitStatus := 0
done := make(chan error, 1)
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Logf("prometheus agent should not be running: %v", err)
actualExitStatus = prom.ProcessState.ExitCode()
case <-time.After(5 * time.Second):
prom.Process.Kill()
}
require.Equal(t, 3, actualExitStatus)
// Assert on last line.
lines := strings.Split(output.String(), "\n")
last := lines[len(lines)-1]
require.Equal(t, "The following flag(s) can not be used in agent mode: [\"--storage.tsdb.path\"]", last)
}
func TestAgentFailedStartupWithInvalidConfig(t *testing.T) {
prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--config.file="+promConfig)
require.NoError(t, prom.Start())
actualExitStatus := 0
done := make(chan error, 1)
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Logf("prometheus agent should not be running: %v", err)
actualExitStatus = prom.ProcessState.ExitCode()
case <-time.After(5 * time.Second):
prom.Process.Kill()
}
require.Equal(t, 2, actualExitStatus)
}
func TestModeSpecificFlags(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
testcases := []struct {
mode string
arg string
exitStatus int
}{
{"agent", "--storage.agent.path", 0},
{"server", "--storage.tsdb.path", 0},
{"server", "--storage.agent.path", 3},
{"agent", "--storage.tsdb.path", 3},
}
for _, tc := range testcases {
t.Run(fmt.Sprintf("%s mode with option %s", tc.mode, tc.arg), func(t *testing.T) {
args := []string{"-test.main", tc.arg, t.TempDir()}
if tc.mode == "agent" {
args = append(args, "--enable-feature=agent", "--config.file="+agentConfig)
} else {
args = append(args, "--config.file="+promConfig)
}
prom := exec.Command(promPath, args...)
// Log stderr in case of failure.
stderr, err := prom.StderrPipe()
require.NoError(t, err)
go func() {
slurp, _ := ioutil.ReadAll(stderr)
t.Log(string(slurp))
}()
err = prom.Start()
require.NoError(t, err)
if tc.exitStatus == 0 {
done := make(chan error, 1)
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Errorf("prometheus should be still running: %v", err)
case <-time.After(5 * time.Second):
prom.Process.Kill()
<-done
}
return
}
err = prom.Wait()
require.Error(t, err)
if exitError, ok := err.(*exec.ExitError); ok {
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, tc.exitStatus, status.ExitStatus())
} else {
t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
}
})
}
}

19
cmd/prometheus/main_unix_test.go
View file

@ -17,11 +17,14 @@
package main
import (
"fmt"
"net/http"
"os"
"os/exec"
"testing"
"time"
"github.com/prometheus/prometheus/util/testutil"
)
// As soon as prometheus starts responding to http request it should be able to
@ -31,11 +34,12 @@ func TestStartupInterrupt(t *testing.T) {
t.Skip("skipping test in short mode.")
}
prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+promData)
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+t.TempDir(), "--web.listen-address=0.0.0.0"+port)
err := prom.Start()
if err != nil {
t.Errorf("execution error: %v", err)
return
t.Fatalf("execution error: %v", err)
}
done := make(chan error, 1)
@ -46,11 +50,13 @@ func TestStartupInterrupt(t *testing.T) {
var startedOk bool
var stoppedErr error
url := "http://localhost" + port + "/graph"
Loop:
for x := 0; x < 10; x++ {
// error=nil means prometheus has started so we can send the interrupt
// signal and wait for the graceful shutdown.
if _, err := http.Get("http://localhost:9090/graph"); err == nil {
if _, err := http.Get(url); err == nil {
startedOk = true
prom.Process.Signal(os.Interrupt)
select {
@ -64,12 +70,11 @@ Loop:
}
if !startedOk {
t.Errorf("prometheus didn't start in the specified timeout")
return
t.Fatal("prometheus didn't start in the specified timeout")
}
if err := prom.Process.Kill(); err == nil {
t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
} else if stoppedErr != nil && stoppedErr.Error() != "signal: interrupt" { // TODO - find a better way to detect when the process didn't exit as expected!
t.Errorf("prometheus exited with an unexpected error:%v", stoppedErr)
t.Errorf("prometheus exited with an unexpected error: %v", stoppedErr)
}
}

5
cmd/prometheus/query_log_test.go
View file

@ -31,6 +31,8 @@ import (
"time"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/util/testutil"
)
type origin int
@ -412,7 +414,6 @@ func TestQueryLog(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
port := 15000
for _, host := range []string{"127.0.0.1", "[::1]"} {
for _, prefix := range []string{"", "/foobar"} {
for _, enabledAtStart := range []bool{true, false} {
@ -422,7 +423,7 @@ func TestQueryLog(t *testing.T) {
host: host,
enabledAtStart: enabledAtStart,
prefix: prefix,
port: port,
port: testutil.RandomUnprivilegedPort(t),
cwd: cwd,
}

2
cmd/promtool/archive.go
View file

@ -21,7 +21,7 @@ import (
"github.com/pkg/errors"
)
const filePerm = 0666
const filePerm = 0o666
type tarGzFileWriter struct {
tarWriter *tar.Writer

15
cmd/promtool/backfill.go
View file

@ -21,8 +21,9 @@ import (
"github.com/go-kit/log"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/textparse"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/textparse"
"github.com/prometheus/prometheus/tsdb"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
)
@ -66,7 +67,7 @@ func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) {
return maxt, mint, nil
}
func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) {
func getCompatibleBlockDuration(maxBlockDuration int64) int64 {
blockDuration := tsdb.DefaultBlockDuration
if maxBlockDuration > tsdb.DefaultBlockDuration {
ranges := tsdb.ExponentialBlockRanges(tsdb.DefaultBlockDuration, 10, 3)
@ -79,6 +80,11 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
}
blockDuration = ranges[idx]
}
return blockDuration
}
func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) {
blockDuration := getCompatibleBlockDuration(maxBlockDuration)
mint = blockDuration * (mint / blockDuration)
db, err := tsdb.OpenDBReadOnly(outputDir, nil)
@ -100,7 +106,6 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
// The next sample is not in this timerange, we can avoid parsing
// the file for this timerange.
continue
}
nextSampleTs = math.MaxInt64
@ -202,13 +207,11 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
return nil
}()
if err != nil {
return errors.Wrap(err, "process blocks")
}
}
return nil
}
func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) {

5
cmd/promtool/backfill_test.go
View file

@ -22,10 +22,11 @@ import (
"testing"
"time"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/stretchr/testify/require"
)
type backfillSample struct {

1
cmd/promtool/debug.go
View file

@ -57,7 +57,6 @@ func debugWrite(cfg debugWriterConfig) error {
return errors.Wrap(err, "error writing into the archive")
}
}
}
if err := archiver.close(); err != nil {

129
cmd/promtool/main.go
View file

@ -44,13 +44,16 @@ import (
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/file"
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
"github.com/prometheus/prometheus/discovery/kubernetes"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/rulefmt"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/rulefmt"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/scrape"
)
func main() {
@ -60,6 +63,11 @@ func main() {
checkCmd := app.Command("check", "Check the resources for validity.")
sdCheckCmd := checkCmd.Command("service-discovery", "Perform service discovery for the given job name and report the results, including relabeling.")
sdConfigFile := sdCheckCmd.Arg("config-file", "The prometheus config file.").Required().ExistingFile()
sdJobName := sdCheckCmd.Arg("job", "The job to run service discovery for.").Required().String()
sdTimeout := sdCheckCmd.Flag("timeout", "The time to wait for discovery results.").Default("30s").Duration()
checkConfigCmd := checkCmd.Command("config", "Check if the config files are valid or not.")
configFiles := checkConfigCmd.Arg(
"config-files",
@ -79,6 +87,7 @@ func main() {
).Required().ExistingFiles()
checkMetricsCmd := checkCmd.Command("metrics", checkMetricsUsage)
agentMode := checkConfigCmd.Flag("agent", "Check config file for Prometheus in Agent mode.").Bool()
queryCmd := app.Command("query", "Run query against a Prometheus server.")
queryCmdFmt := queryCmd.Flag("format", "Output format of the query.").Short('o').Default("promql").Enum("promql", "json")
@ -198,8 +207,11 @@ func main() {
}
switch parsedCmd {
case sdCheckCmd.FullCommand():
os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout))
case checkConfigCmd.FullCommand():
os.Exit(CheckConfig(*configFiles...))
os.Exit(CheckConfig(*agentMode, *configFiles...))
case checkWebConfigCmd.FullCommand():
os.Exit(CheckWebConfig(*webConfigFiles...))
@ -245,21 +257,21 @@ func main() {
case tsdbDumpCmd.FullCommand():
os.Exit(checkErr(dumpSamples(*dumpPath, *dumpMinTime, *dumpMaxTime)))
//TODO(aSquare14): Work on adding support for custom block size.
// TODO(aSquare14): Work on adding support for custom block size.
case openMetricsImportCmd.FullCommand():
os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration))
case importRulesCmd.FullCommand():
os.Exit(checkErr(importRules(*importRulesURL, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *importRulesFiles...)))
os.Exit(checkErr(importRules(*importRulesURL, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...)))
}
}
// CheckConfig validates configuration files.
func CheckConfig(files ...string) int {
func CheckConfig(agentMode bool, files ...string) int {
failed := false
for _, f := range files {
ruleFiles, err := checkConfig(f)
ruleFiles, err := checkConfig(agentMode, f)
if err != nil {
fmt.Fprintln(os.Stderr, " FAILED:", err)
failed = true
@ -314,10 +326,10 @@ func checkFileExists(fn string) error {
return err
}
func checkConfig(filename string) ([]string, error) {
func checkConfig(agentMode bool, filename string) ([]string, error) {
fmt.Println("Checking", filename)
cfg, err := config.LoadFile(filename, false, log.NewNopLogger())
cfg, err := config.LoadFile(filename, agentMode, false, log.NewNopLogger())
if err != nil {
return nil, err
}
@ -363,19 +375,60 @@ func checkConfig(filename string) ([]string, error) {
}
if len(files) != 0 {
for _, f := range files {
err = checkSDFile(f)
var targetGroups []*targetgroup.Group
targetGroups, err = checkSDFile(f)
if err != nil {
return nil, errors.Errorf("checking SD file %q: %v", file, err)
}
if err := checkTargetGroupsForScrapeConfig(targetGroups, scfg); err != nil {
return nil, err
}
}
continue
}
fmt.Printf(" WARNING: file %q for file_sd in scrape job %q does not exist\n", file, scfg.JobName)
}
case discovery.StaticConfig:
if err := checkTargetGroupsForScrapeConfig(c, scfg); err != nil {
return nil, err
}
}
}
}
alertConfig := cfg.AlertingConfig
for _, amcfg := range alertConfig.AlertmanagerConfigs {
for _, c := range amcfg.ServiceDiscoveryConfigs {
switch c := c.(type) {
case *file.SDConfig:
for _, file := range c.Files {
files, err := filepath.Glob(file)
if err != nil {
return nil, err
}
if len(files) != 0 {
for _, f := range files {
var targetGroups []*targetgroup.Group
targetGroups, err = checkSDFile(f)
if err != nil {
return nil, errors.Errorf("checking SD file %q: %v", file, err)
}
if err := checkTargetGroupsForAlertmanager(targetGroups, amcfg); err != nil {
return nil, err
}
}
continue
}
fmt.Printf(" WARNING: file %q for file_sd in alertmanager config does not exist\n", file)
}
case discovery.StaticConfig:
if err := checkTargetGroupsForAlertmanager(c, amcfg); err != nil {
return nil, err
}
}
}
}
return ruleFiles, nil
}
@ -397,16 +450,16 @@ func checkTLSConfig(tlsConfig config_util.TLSConfig) error {
return nil
}
func checkSDFile(filename string) error {
func checkSDFile(filename string) ([]*targetgroup.Group, error) {
fd, err := os.Open(filename)
if err != nil {
return err
return nil, err
}
defer fd.Close()
content, err := ioutil.ReadAll(fd)
if err != nil {
return err
return nil, err
}
var targetGroups []*targetgroup.Group
@ -414,23 +467,23 @@ func checkSDFile(filename string) error {
switch ext := filepath.Ext(filename); strings.ToLower(ext) {
case ".json":
if err := json.Unmarshal(content, &targetGroups); err != nil {
return err
return nil, err
}
case ".yml", ".yaml":
if err := yaml.UnmarshalStrict(content, &targetGroups); err != nil {
return err
return nil, err
}
default:
return errors.Errorf("invalid file extension: %q", ext)
return nil, errors.Errorf("invalid file extension: %q", ext)
}
for i, tg := range targetGroups {
if tg == nil {
return errors.Errorf("nil target group item found (index %d)", i)
return nil, errors.Errorf("nil target group item found (index %d)", i)
}
}
return nil
return targetGroups, nil
}
// CheckRules validates rule files.
@ -507,7 +560,6 @@ func checkDuplicates(groups []rulefmt.RuleGroup) []compareRuleType {
var rules compareRuleTypes
for _, group := range groups {
for _, rule := range group.Rules {
rules = append(rules, compareRuleType{
metric: ruleMetric(rule),
@ -721,7 +773,7 @@ func QuerySeries(url *url.URL, matchers []string, start, end string, p printer)
}
// QueryLabels queries for label values against a Prometheus server.
func QueryLabels(url *url.URL, name string, start, end string, p printer) int {
func QueryLabels(url *url.URL, name, start, end string, p printer) int {
if url.Scheme == "" {
url.Scheme = "http"
}
@ -899,11 +951,13 @@ type promqlPrinter struct{}
func (p *promqlPrinter) printValue(v model.Value) {
fmt.Println(v)
}
func (p *promqlPrinter) printSeries(val []model.LabelSet) {
for _, v := range val {
fmt.Println(v)
}
}
func (p *promqlPrinter) printLabelValues(val model.LabelValues) {
for _, v := range val {
fmt.Println(v)
@ -916,10 +970,12 @@ func (j *jsonPrinter) printValue(v model.Value) {
//nolint:errcheck
json.NewEncoder(os.Stdout).Encode(v)
}
func (j *jsonPrinter) printSeries(v []model.LabelSet) {
//nolint:errcheck
json.NewEncoder(os.Stdout).Encode(v)
}
func (j *jsonPrinter) printLabelValues(v model.LabelValues) {
//nolint:errcheck
json.NewEncoder(os.Stdout).Encode(v)
@ -927,7 +983,7 @@ func (j *jsonPrinter) printLabelValues(v model.LabelValues) {
// importRules backfills recording rules from the files provided. The output are blocks of data
// at the outputDir location.
func importRules(url *url.URL, start, end, outputDir string, evalInterval time.Duration, files ...string) error {
func importRules(url *url.URL, start, end, outputDir string, evalInterval, maxBlockDuration time.Duration, files ...string) error {
ctx := context.Background()
var stime, etime time.Time
var err error
@ -950,10 +1006,11 @@ func importRules(url *url.URL, start, end, outputDir string, evalInterval time.D
}
cfg := ruleImporterConfig{
outputDir: outputDir,
start: stime,
end: etime,
evalInterval: evalInterval,
outputDir: outputDir,
start: stime,
end: etime,
evalInterval: evalInterval,
maxBlockDuration: maxBlockDuration,
}
client, err := api.NewClient(api.Config{
Address: url.String(),
@ -980,3 +1037,25 @@ func importRules(url *url.URL, start, end, outputDir string, evalInterval time.D
return nil
}
func checkTargetGroupsForAlertmanager(targetGroups []*targetgroup.Group, amcfg *config.AlertmanagerConfig) error {
for _, tg := range targetGroups {
if _, _, err := notifier.AlertmanagerFromGroup(tg, amcfg); err != nil {
return err
}
}
return nil
}
func checkTargetGroupsForScrapeConfig(targetGroups []*targetgroup.Group, scfg *config.ScrapeConfig) error {
for _, tg := range targetGroups {
_, failures := scrape.TargetsFromGroup(tg, scfg)
if len(failures) > 0 {
first := failures[0]
return first
}
}
return nil
}

46
cmd/promtool/main_test.go
View file

@ -21,9 +21,10 @@ import (
"testing"
"time"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/rulefmt"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/rulefmt"
)
func TestQueryRange(t *testing.T) {
@ -111,7 +112,7 @@ func TestCheckSDFile(t *testing.T) {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
err := checkSDFile(test.file)
_, err := checkSDFile(test.file)
if test.err != "" {
require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error())
return
@ -163,3 +164,42 @@ func BenchmarkCheckDuplicates(b *testing.B) {
checkDuplicates(rgs.Groups)
}
}
func TestCheckTargetConfig(t *testing.T) {
cases := []struct {
name string
file string
err string
}{
{
name: "url_in_scrape_targetgroup_with_relabel_config.good",
file: "url_in_scrape_targetgroup_with_relabel_config.good.yml",
err: "",
},
{
name: "url_in_alert_targetgroup_with_relabel_config.good",
file: "url_in_alert_targetgroup_with_relabel_config.good.yml",
err: "",
},
{
name: "url_in_scrape_targetgroup_with_relabel_config.bad",
file: "url_in_scrape_targetgroup_with_relabel_config.bad.yml",
err: "instance 0 in group 0: \"http://bad\" is not a valid hostname",
},
{
name: "url_in_alert_targetgroup_with_relabel_config.bad",
file: "url_in_alert_targetgroup_with_relabel_config.bad.yml",
err: "\"http://bad\" is not a valid hostname",
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, err := checkConfig(false, "testdata/"+test.file)
if test.err != "" {
require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error())
return
}
require.NoError(t, err)
})
}
}

25
cmd/promtool/rules.go
View file

@ -23,8 +23,9 @@ import (
"github.com/pkg/errors"
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/timestamp"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/timestamp"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
@ -48,10 +49,11 @@ type ruleImporter struct {
}
type ruleImporterConfig struct {
outputDir string
start time.Time
end time.Time
evalInterval time.Duration
outputDir string
start time.Time
end time.Time
evalInterval time.Duration
maxBlockDuration time.Duration
}
// newRuleImporter creates a new rule importer that can be used to parse and evaluate recording rule files and create new series
@ -83,7 +85,7 @@ func (importer *ruleImporter) importAll(ctx context.Context) (errs []error) {
for i, r := range group.Rules() {
level.Info(importer.logger).Log("backfiller", "processing rule", "id", i, "name", r.Name())
if err := importer.importRule(ctx, r.Query().String(), r.Name(), r.Labels(), importer.config.start, importer.config.end, group); err != nil {
if err := importer.importRule(ctx, r.Query().String(), r.Name(), r.Labels(), importer.config.start, importer.config.end, int64(importer.config.maxBlockDuration/time.Millisecond), group); err != nil {
errs = append(errs, err)
}
}
@ -92,8 +94,9 @@ func (importer *ruleImporter) importAll(ctx context.Context) (errs []error) {
}
// importRule queries a prometheus API to evaluate rules at times in the past.
func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName string, ruleLabels labels.Labels, start, end time.Time, grp *rules.Group) (err error) {
blockDuration := tsdb.DefaultBlockDuration
func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName string, ruleLabels labels.Labels, start, end time.Time,
maxBlockDuration int64, grp *rules.Group) (err error) {
blockDuration := getCompatibleBlockDuration(maxBlockDuration)
startInMs := start.Unix() * int64(time.Second/time.Millisecond)
endInMs := end.Unix() * int64(time.Second/time.Millisecond)
@ -130,7 +133,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName
// also need to append samples throughout the whole block range. To allow that, we
// pretend that the block is twice as large here, but only really add sample in the
// original interval later.
w, err := tsdb.NewBlockWriter(log.NewNopLogger(), importer.config.outputDir, 2*tsdb.DefaultBlockDuration)
w, err := tsdb.NewBlockWriter(log.NewNopLogger(), importer.config.outputDir, 2*blockDuration)
if err != nil {
return errors.Wrap(err, "new block writer")
}
@ -168,7 +171,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName
}
}
default:
return errors.New(fmt.Sprintf("rule result is wrong type %s", val.Type().String()))
return fmt.Errorf("rule result is wrong type %s", val.Type().String())
}
if err := app.flushAndCommit(ctx); err != nil {

48
cmd/promtool/rules_test.go
View file

@ -25,9 +25,10 @@ import (
"github.com/go-kit/log"
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/tsdb"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb"
)
type mockQueryRangeAPI struct {
@ -38,6 +39,8 @@ func (mockAPI mockQueryRangeAPI) QueryRange(ctx context.Context, query string, r
return mockAPI.samples, v1.Warnings{}, nil
}
const defaultBlockDuration = time.Duration(tsdb.DefaultBlockDuration) * time.Millisecond
// TestBackfillRuleIntegration is an integration test that runs all the rule importer code to confirm the parts work together.
func TestBackfillRuleIntegration(t *testing.T) {
const (
@ -46,23 +49,26 @@ func TestBackfillRuleIntegration(t *testing.T) {
testValue2 = 98
)
var (
start = time.Date(2009, time.November, 10, 6, 34, 0, 0, time.UTC)
testTime = model.Time(start.Add(-9 * time.Hour).Unix())
testTime2 = model.Time(start.Add(-8 * time.Hour).Unix())
start = time.Date(2009, time.November, 10, 6, 34, 0, 0, time.UTC)
testTime = model.Time(start.Add(-9 * time.Hour).Unix())
testTime2 = model.Time(start.Add(-8 * time.Hour).Unix())
twentyFourHourDuration, _ = time.ParseDuration("24h")
)
var testCases = []struct {
testCases := []struct {
name string
runcount int
maxBlockDuration time.Duration
expectedBlockCount int
expectedSeriesCount int
expectedSampleCount int
samples []*model.SampleStream
}{
{"no samples", 1, 0, 0, 0, []*model.SampleStream{}},
{"run importer once", 1, 8, 4, 4, []*model.SampleStream{{Metric: model.Metric{"name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}}}}},
{"run importer with dup name label", 1, 8, 4, 4, []*model.SampleStream{{Metric: model.Metric{"__name__": "val1", "name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}}}}},
{"one importer twice", 2, 8, 4, 8, []*model.SampleStream{{Metric: model.Metric{"name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}, {Timestamp: testTime2, Value: testValue2}}}}},
{"no samples", 1, defaultBlockDuration, 0, 0, 0, []*model.SampleStream{}},
{"run importer once", 1, defaultBlockDuration, 8, 4, 4, []*model.SampleStream{{Metric: model.Metric{"name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}}}}},
{"run importer with dup name label", 1, defaultBlockDuration, 8, 4, 4, []*model.SampleStream{{Metric: model.Metric{"__name__": "val1", "name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}}}}},
{"one importer twice", 2, defaultBlockDuration, 8, 4, 8, []*model.SampleStream{{Metric: model.Metric{"name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}, {Timestamp: testTime2, Value: testValue2}}}}},
{"run importer once with larger blocks", 1, twentyFourHourDuration, 4, 4, 4, []*model.SampleStream{{Metric: model.Metric{"name1": "val1"}, Values: []model.SamplePair{{Timestamp: testTime, Value: testValue}}}}},
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
@ -76,7 +82,8 @@ func TestBackfillRuleIntegration(t *testing.T) {
// Execute the test more than once to simulate running the rule importer twice with the same data.
// We expect duplicate blocks with the same series are created when run more than once.
for i := 0; i < tt.runcount; i++ {
ruleImporter, err := newTestRuleImporter(ctx, start, tmpDir, tt.samples)
ruleImporter, err := newTestRuleImporter(ctx, start, tmpDir, tt.samples, tt.maxBlockDuration)
require.NoError(t, err)
path1 := filepath.Join(tmpDir, "test.file")
require.NoError(t, createSingleRuleTestFiles(path1))
@ -162,13 +169,14 @@ func TestBackfillRuleIntegration(t *testing.T) {
}
}
func newTestRuleImporter(ctx context.Context, start time.Time, tmpDir string, testSamples model.Matrix) (*ruleImporter, error) {
func newTestRuleImporter(ctx context.Context, start time.Time, tmpDir string, testSamples model.Matrix, maxBlockDuration time.Duration) (*ruleImporter, error) {
logger := log.NewNopLogger()
cfg := ruleImporterConfig{
outputDir: tmpDir,
start: start.Add(-10 * time.Hour),
end: start.Add(-7 * time.Hour),
evalInterval: 60 * time.Second,
outputDir: tmpDir,
start: start.Add(-10 * time.Hour),
end: start.Add(-7 * time.Hour),
evalInterval: 60 * time.Second,
maxBlockDuration: maxBlockDuration,
}
return newRuleImporter(logger, cfg, mockQueryRangeAPI{
@ -185,7 +193,7 @@ func createSingleRuleTestFiles(path string) error {
labels:
testlabel11: testlabelvalue11
`
return ioutil.WriteFile(path, []byte(recordingRules), 0777)
return ioutil.WriteFile(path, []byte(recordingRules), 0o777)
}
func createMultiRuleTestFiles(path string) error {
@ -205,7 +213,7 @@ func createMultiRuleTestFiles(path string) error {
labels:
testlabel11: testlabelvalue13
`
return ioutil.WriteFile(path, []byte(recordingRules), 0777)
return ioutil.WriteFile(path, []byte(recordingRules), 0o777)
}
// TestBackfillLabels confirms that the labels in the rule file override the labels from the metrics
@ -225,7 +233,7 @@ func TestBackfillLabels(t *testing.T) {
Values: []model.SamplePair{{Timestamp: model.TimeFromUnixNano(start.UnixNano()), Value: 123}},
},
}
ruleImporter, err := newTestRuleImporter(ctx, start, tmpDir, mockAPISamples)
ruleImporter, err := newTestRuleImporter(ctx, start, tmpDir, mockAPISamples, defaultBlockDuration)
require.NoError(t, err)
path := filepath.Join(tmpDir, "test.file")
@ -237,7 +245,7 @@ func TestBackfillLabels(t *testing.T) {
labels:
name1: value-from-rule
`
require.NoError(t, ioutil.WriteFile(path, []byte(recordingRules), 0777))
require.NoError(t, ioutil.WriteFile(path, []byte(recordingRules), 0o777))
errs := ruleImporter.loadGroups(ctx, []string{path})
for _, err := range errs {
require.NoError(t, err)

148
cmd/promtool/sd.go Normal file
View file

@ -0,0 +1,148 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"reflect"
"time"
"github.com/go-kit/log"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/scrape"
)
type sdCheckResult struct {
DiscoveredLabels labels.Labels `json:"discoveredLabels"`
Labels labels.Labels `json:"labels"`
Error error `json:"error,omitempty"`
}
// CheckSD performs service discovery for the given job name and reports the results.
func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration) int {
logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
cfg, err := config.LoadFile(sdConfigFiles, false, false, logger)
if err != nil {
fmt.Fprintln(os.Stderr, "Cannot load config", err)
return 2
}
var scrapeConfig *config.ScrapeConfig
jobs := []string{}
jobMatched := false
for _, v := range cfg.ScrapeConfigs {
jobs = append(jobs, v.JobName)
if v.JobName == sdJobName {
jobMatched = true
scrapeConfig = v
break
}
}
if !jobMatched {
fmt.Fprintf(os.Stderr, "Job %s not found. Select one of:\n", sdJobName)
for _, job := range jobs {
fmt.Fprintf(os.Stderr, "\t%s\n", job)
}
return 1
}
targetGroupChan := make(chan []*targetgroup.Group)
ctx, cancel := context.WithTimeout(context.Background(), sdTimeout)
defer cancel()
for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger})
if err != nil {
fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
return 2
}
go d.Run(ctx, targetGroupChan)
}
var targetGroups []*targetgroup.Group
sdCheckResults := make(map[string][]*targetgroup.Group)
outerLoop:
for {
select {
case targetGroups = <-targetGroupChan:
for _, tg := range targetGroups {
sdCheckResults[tg.Source] = append(sdCheckResults[tg.Source], tg)
}
case <-ctx.Done():
break outerLoop
}
}
results := []sdCheckResult{}
for _, tgs := range sdCheckResults {
results = append(results, getSDCheckResult(tgs, scrapeConfig)...)
}
res, err := json.MarshalIndent(results, "", " ")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not marshal result json: %s", err)
return 2
}
fmt.Printf("%s", res)
return 0
}
func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.ScrapeConfig) []sdCheckResult {
sdCheckResults := []sdCheckResult{}
for _, targetGroup := range targetGroups {
for _, target := range targetGroup.Targets {
labelSlice := make([]labels.Label, 0, len(target)+len(targetGroup.Labels))
for name, value := range target {
labelSlice = append(labelSlice, labels.Label{Name: string(name), Value: string(value)})
}
for name, value := range targetGroup.Labels {
if _, ok := target[name]; !ok {
labelSlice = append(labelSlice, labels.Label{Name: string(name), Value: string(value)})
}
}
targetLabels := labels.New(labelSlice...)
res, orig, err := scrape.PopulateLabels(targetLabels, scrapeConfig)
result := sdCheckResult{
DiscoveredLabels: orig,
Labels: res,
Error: err,
}
duplicateRes := false
for _, sdCheckRes := range sdCheckResults {
if reflect.DeepEqual(sdCheckRes, result) {
duplicateRes = true
break
}
}
if !duplicateRes {
sdCheckResults = append(sdCheckResults, result)
}
}
}
return sdCheckResults
}

70
cmd/promtool/sd_test.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"testing"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/stretchr/testify/require"
)
func TestSDCheckResult(t *testing.T) {
targetGroups := []*targetgroup.Group{{
Targets: []model.LabelSet{
map[model.LabelName]model.LabelValue{"__address__": "localhost:8080", "foo": "bar"},
},
}}
reg, err := relabel.NewRegexp("(.*)")
require.Nil(t, err)
scrapeConfig := &config.ScrapeConfig{
RelabelConfigs: []*relabel.Config{{
SourceLabels: model.LabelNames{"foo"},
Action: relabel.Replace,
TargetLabel: "newfoo",
Regex: reg,
Replacement: "$1",
}},
}
expectedSDCheckResult := []sdCheckResult{
{
DiscoveredLabels: labels.Labels{
labels.Label{Name: "__address__", Value: "localhost:8080"},
labels.Label{Name: "__scrape_interval__", Value: "0s"},
labels.Label{Name: "__scrape_timeout__", Value: "0s"},
labels.Label{Name: "foo", Value: "bar"},
},
Labels: labels.Labels{
labels.Label{Name: "__address__", Value: "localhost:8080"},
labels.Label{Name: "__scrape_interval__", Value: "0s"},
labels.Label{Name: "__scrape_timeout__", Value: "0s"},
labels.Label{Name: "foo", Value: "bar"},
labels.Label{Name: "instance", Value: "localhost:8080"},
labels.Label{Name: "newfoo", Value: "bar"},
},
Error: nil,
},
}
require.Equal(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig))
}

8
cmd/promtool/testdata/url_in_alert_targetgroup_with_relabel_config.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
alerting:
alertmanagers:
- relabel_configs:
- source_labels: [__address__]
target_label: __param_target
static_configs:
- targets:
- http://bad

10
cmd/promtool/testdata/url_in_alert_targetgroup_with_relabel_config.good.yml vendored Normal file
View file

@ -0,0 +1,10 @@
alerting:
alertmanagers:
- relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: good
static_configs:
- targets:
- http://bad

8
cmd/promtool/testdata/url_in_scrape_targetgroup_with_relabel_config.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
static_configs:
- targets:
- http://bad

10
cmd/promtool/testdata/url_in_scrape_targetgroup_with_relabel_config.good.yml vendored Normal file
View file

@ -0,0 +1,10 @@
scrape_configs:
- job_name: prometheus
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: good
static_configs:
- targets:
- http://good

18
cmd/promtool/tsdb.go
View file

@ -17,7 +17,6 @@ import (
"bufio"
"context"
"fmt"
"github.com/prometheus/prometheus/tsdb/index"
"io"
"io/ioutil"
"math"
@ -32,11 +31,14 @@ import (
"text/tabwriter"
"time"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/alecthomas/units"
"github.com/go-kit/log"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunks"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
@ -78,7 +80,7 @@ func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) err
if err := os.RemoveAll(b.outPath); err != nil {
return err
}
if err := os.MkdirAll(b.outPath, 0777); err != nil {
if err := os.MkdirAll(b.outPath, 0o777); err != nil {
return err
}
@ -187,7 +189,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in
type sample struct {
labels labels.Labels
value int64
ref *uint64
ref *storage.SeriesRef
}
scrape := make([]*sample, 0, len(lbls))
@ -207,7 +209,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in
for _, s := range scrape {
s.value += 1000
var ref uint64
var ref storage.SeriesRef
if s.ref != nil {
ref = *s.ref
}
@ -589,7 +591,7 @@ func analyzeCompaction(block tsdb.BlockReader, indexr tsdb.IndexReader) (err err
histogram := make([]int, nBuckets)
totalChunks := 0
for postingsr.Next() {
var lbsl = labels.Labels{}
lbsl := labels.Labels{}
var chks []chunks.Meta
if err := indexr.Series(postingsr.At(), &lbsl, &chks); err != nil {
return err
@ -671,14 +673,14 @@ func checkErr(err error) int {
return 0
}
func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) int {
func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) int {
inputFile, err := fileutil.OpenMmapFile(path)
if err != nil {
return checkErr(err)
}
defer inputFile.Close()
if err := os.MkdirAll(outputDir, 0777); err != nil {
if err := os.MkdirAll(outputDir, 0o777); err != nil {
return checkErr(errors.Wrap(err, "create output dir"))
}

65
cmd/promtool/unittest.go
View file

@ -30,7 +30,7 @@ import (
"github.com/prometheus/common/model"
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/prometheus/prometheus/rules"
@ -47,6 +47,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int {
fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error())
fmt.Println()
}
failed = true
} else {
@ -313,30 +314,18 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
})
}
var sb strings.Builder
if gotAlerts.Len() != expAlerts.Len() {
sort.Sort(gotAlerts)
sort.Sort(expAlerts)
if !reflect.DeepEqual(expAlerts, gotAlerts) {
var testName string
if tg.TestGroupName != "" {
fmt.Fprintf(&sb, " name: %s,\n", tg.TestGroupName)
}
fmt.Fprintf(&sb, " alertname:%s, time:%s, \n", testcase.Alertname, testcase.EvalTime.String())
fmt.Fprintf(&sb, " exp:%#v, \n", expAlerts.String())
fmt.Fprintf(&sb, " got:%#v", gotAlerts.String())
errs = append(errs, errors.New(sb.String()))
} else {
sort.Sort(gotAlerts)
sort.Sort(expAlerts)
if !reflect.DeepEqual(expAlerts, gotAlerts) {
if tg.TestGroupName != "" {
fmt.Fprintf(&sb, " name: %s,\n", tg.TestGroupName)
}
fmt.Fprintf(&sb, " alertname:%s, time:%s, \n", testcase.Alertname, testcase.EvalTime.String())
fmt.Fprintf(&sb, " exp:%#v, \n", expAlerts.String())
fmt.Fprintf(&sb, " got:%#v", gotAlerts.String())
errs = append(errs, errors.New(sb.String()))
testName = fmt.Sprintf(" name: %s,\n", tg.TestGroupName)
}
expString := indentLines(expAlerts.String(), " ")
gotString := indentLines(gotAlerts.String(), " ")
errs = append(errs, errors.Errorf("%s alertname: %s, time: %s, \n exp:%v, \n got:%v",
testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
}
}
@ -385,7 +374,7 @@ Outer:
return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
})
if !reflect.DeepEqual(expSamples, gotSamples) {
errs = append(errs, errors.Errorf(" expr: %q, time: %s,\n exp:%#v\n got:%#v", testCase.Expr,
errs = append(errs, errors.Errorf(" expr: %q, time: %s,\n exp: %v\n got: %v", testCase.Expr,
testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
}
}
@ -398,7 +387,6 @@ Outer:
// seriesLoadingString returns the input series in PromQL notation.
func (tg *testGroup) seriesLoadingString() string {
result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
for _, is := range tg.InputSeries {
result += fmt.Sprintf(" %v %v\n", is.Series, is.Values)
@ -468,6 +456,23 @@ func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, q
}
}
// indentLines prefixes each line in the supplied string with the given "indent"
// string.
func indentLines(lines, indent string) string {
sb := strings.Builder{}
n := strings.Split(lines, "\n")
for i, l := range n {
if i > 0 {
sb.WriteString(indent)
}
sb.WriteString(l)
if i != len(n)-1 {
sb.WriteRune('\n')
}
}
return sb.String()
}
type labelsAndAnnotations []labelAndAnnotation
func (la labelsAndAnnotations) Len() int { return len(la) }
@ -484,11 +489,11 @@ func (la labelsAndAnnotations) String() string {
if len(la) == 0 {
return "[]"
}
s := "[" + la[0].String()
for _, l := range la[1:] {
s += ", " + l.String()
s := "[\n0:" + indentLines("\n"+la[0].String(), " ")
for i, l := range la[1:] {
s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), " ")
}
s += "]"
s += "\n]"
return s
}
@ -499,7 +504,7 @@ type labelAndAnnotation struct {
}
func (la *labelAndAnnotation) String() string {
return "Labels:" + la.Labels.String() + " Annotations:" + la.Annotations.String()
return "Labels:" + la.Labels.String() + "\nAnnotations:" + la.Annotations.String()
}
type series struct {

27
config/config.go
View file

@ -33,8 +33,8 @@ import (
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/relabel"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
)
var (
@ -99,7 +99,7 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro
}
// LoadFile parses the given YAML file into a Config.
func LoadFile(filename string, expandExternalLabels bool, logger log.Logger) (*Config, error) {
func LoadFile(filename string, agentMode, expandExternalLabels bool, logger log.Logger) (*Config, error) {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
@ -108,6 +108,25 @@ func LoadFile(filename string, expandExternalLabels bool, logger log.Logger) (*C
if err != nil {
return nil, errors.Wrapf(err, "parsing YAML file %s", filename)
}
if agentMode {
if len(cfg.RemoteWriteConfigs) == 0 {
return nil, errors.New("at least one remote_write target must be specified in agent mode")
}
if len(cfg.AlertingConfig.AlertmanagerConfigs) > 0 || len(cfg.AlertingConfig.AlertRelabelConfigs) > 0 {
return nil, errors.New("field alerting is not allowed in agent mode")
}
if len(cfg.RuleFiles) > 0 {
return nil, errors.New("field rule_files is not allowed in agent mode")
}
if len(cfg.RemoteReadConfigs) > 0 {
return nil, errors.New("field remote_read is not allowed in agent mode")
}
}
cfg.SetDirectory(filepath.Dir(filename))
return cfg, nil
}
@ -169,7 +188,7 @@ var (
// Backoff times for retrying a batch of samples on recoverable errors.
MinBackoff: model.Duration(30 * time.Millisecond),
MaxBackoff: model.Duration(100 * time.Millisecond),
MaxBackoff: model.Duration(5 * time.Second),
}
// DefaultMetadataConfig is the default metadata configuration for a remote write endpoint.

258
config/config_test.go
View file

@ -52,8 +52,8 @@ import (
"github.com/prometheus/prometheus/discovery/uyuni"
"github.com/prometheus/prometheus/discovery/xds"
"github.com/prometheus/prometheus/discovery/zookeeper"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/relabel"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
)
func mustParseURL(u string) *config.URL {
@ -103,6 +103,10 @@ var expectedConf = &Config{
ClientID: "123",
ClientSecret: "456",
TokenURL: "http://remote1/auth",
TLSConfig: config.TLSConfig{
CertFile: filepath.FromSlash("testdata/valid_cert_file"),
KeyFile: filepath.FromSlash("testdata/valid_key_file"),
},
},
FollowRedirects: true,
},
@ -565,6 +569,7 @@ var expectedConf = &Config{
AuthenticationMethod: "OAuth",
RefreshInterval: model.Duration(5 * time.Minute),
Port: 9100,
HTTPClientConfig: config.DefaultHTTPClientConfig,
},
},
},
@ -779,17 +784,19 @@ var expectedConf = &Config{
Scheme: DefaultScrapeConfig.Scheme,
HTTPClientConfig: config.DefaultHTTPClientConfig,
ServiceDiscoveryConfigs: discovery.Configs{&openstack.SDConfig{
Role: "instance",
Region: "RegionOne",
Port: 80,
Availability: "public",
RefreshInterval: model.Duration(60 * time.Second),
TLSConfig: config.TLSConfig{
CAFile: "testdata/valid_ca_file",
CertFile: "testdata/valid_cert_file",
KeyFile: "testdata/valid_key_file",
}},
ServiceDiscoveryConfigs: discovery.Configs{
&openstack.SDConfig{
Role: "instance",
Region: "RegionOne",
Port: 80,
Availability: "public",
RefreshInterval: model.Duration(60 * time.Second),
TLSConfig: config.TLSConfig{
CAFile: "testdata/valid_ca_file",
CertFile: "testdata/valid_cert_file",
KeyFile: "testdata/valid_key_file",
},
},
},
},
{
@ -803,22 +810,23 @@ var expectedConf = &Config{
Scheme: DefaultScrapeConfig.Scheme,
HTTPClientConfig: config.DefaultHTTPClientConfig,
ServiceDiscoveryConfigs: discovery.Configs{&puppetdb.SDConfig{
URL: "https://puppetserver/",
Query: "resources { type = \"Package\" and title = \"httpd\" }",
IncludeParameters: true,
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
HTTPClientConfig: config.HTTPClientConfig{
FollowRedirects: true,
TLSConfig: config.TLSConfig{
CAFile: "testdata/valid_ca_file",
CertFile: "testdata/valid_cert_file",
KeyFile: "testdata/valid_key_file",
ServiceDiscoveryConfigs: discovery.Configs{
&puppetdb.SDConfig{
URL: "https://puppetserver/",
Query: "resources { type = \"Package\" and title = \"httpd\" }",
IncludeParameters: true,
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
HTTPClientConfig: config.HTTPClientConfig{
FollowRedirects: true,
TLSConfig: config.TLSConfig{
CAFile: "testdata/valid_ca_file",
CertFile: "testdata/valid_cert_file",
KeyFile: "testdata/valid_key_file",
},
},
},
},
},
},
{
JobName: "hetzner",
@ -981,7 +989,7 @@ var expectedConf = &Config{
}
func TestYAMLRoundtrip(t *testing.T) {
want, err := LoadFile("testdata/roundtrip.good.yml", false, log.NewNopLogger())
want, err := LoadFile("testdata/roundtrip.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
out, err := yaml.Marshal(want)
@ -994,7 +1002,7 @@ func TestYAMLRoundtrip(t *testing.T) {
}
func TestRemoteWriteRetryOnRateLimit(t *testing.T) {
want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml", false, log.NewNopLogger())
want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
out, err := yaml.Marshal(want)
@ -1010,16 +1018,16 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) {
func TestLoadConfig(t *testing.T) {
// Parse a valid file that sets a global scrape timeout. This tests whether parsing
// an overwritten default field in the global config permanently changes the default.
_, err := LoadFile("testdata/global_timeout.good.yml", false, log.NewNopLogger())
_, err := LoadFile("testdata/global_timeout.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
c, err := LoadFile("testdata/conf.good.yml", false, log.NewNopLogger())
c, err := LoadFile("testdata/conf.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, expectedConf, c)
}
func TestScrapeIntervalLarger(t *testing.T) {
c, err := LoadFile("testdata/scrape_interval_larger.good.yml", false, log.NewNopLogger())
c, err := LoadFile("testdata/scrape_interval_larger.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, 1, len(c.ScrapeConfigs))
for _, sc := range c.ScrapeConfigs {
@ -1029,7 +1037,7 @@ func TestScrapeIntervalLarger(t *testing.T) {
// YAML marshaling must not reveal authentication credentials.
func TestElideSecrets(t *testing.T) {
c, err := LoadFile("testdata/conf.good.yml", false, log.NewNopLogger())
c, err := LoadFile("testdata/conf.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
secretRe := regexp.MustCompile(`\\u003csecret\\u003e|<secret>`)
@ -1046,31 +1054,31 @@ func TestElideSecrets(t *testing.T) {
func TestLoadConfigRuleFilesAbsolutePath(t *testing.T) {
// Parse a valid file that sets a rule files with an absolute path
c, err := LoadFile(ruleFilesConfigFile, false, log.NewNopLogger())
c, err := LoadFile(ruleFilesConfigFile, false, false, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, ruleFilesExpectedConf, c)
}
func TestKubernetesEmptyAPIServer(t *testing.T) {
_, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml", false, log.NewNopLogger())
_, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
}
func TestKubernetesWithKubeConfig(t *testing.T) {
_, err := LoadFile("testdata/kubernetes_kubeconfig_without_apiserver.good.yml", false, log.NewNopLogger())
_, err := LoadFile("testdata/kubernetes_kubeconfig_without_apiserver.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
}
func TestKubernetesSelectors(t *testing.T) {
_, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml", false, log.NewNopLogger())
_, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_node.good.yml", false, log.NewNopLogger())
_, err = LoadFile("testdata/kubernetes_selectors_node.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml", false, log.NewNopLogger())
_, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml", false, log.NewNopLogger())
_, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_service.good.yml", false, log.NewNopLogger())
_, err = LoadFile("testdata/kubernetes_selectors_service.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
}
@ -1081,170 +1089,224 @@ var expectedErrors = []struct {
{
filename: "jobname.bad.yml",
errMsg: `job_name is empty`,
}, {
},
{
filename: "jobname_dup.bad.yml",
errMsg: `found multiple scrape configs with job name "prometheus"`,
}, {
},
{
filename: "scrape_interval.bad.yml",
errMsg: `scrape timeout greater than scrape interval`,
}, {
},
{
filename: "labelname.bad.yml",
errMsg: `"not$allowed" is not a valid label name`,
}, {
},
{
filename: "labelname2.bad.yml",
errMsg: `"not:allowed" is not a valid label name`,
}, {
},
{
filename: "labelvalue.bad.yml",
errMsg: `"\xff" is not a valid label value`,
}, {
},
{
filename: "regex.bad.yml",
errMsg: "error parsing regexp",
}, {
},
{
filename: "modulus_missing.bad.yml",
errMsg: "relabel configuration for hashmod requires non-zero modulus",
}, {
},
{
filename: "labelkeep.bad.yml",
errMsg: "labelkeep action requires only 'regex', and no other fields",
}, {
},
{
filename: "labelkeep2.bad.yml",
errMsg: "labelkeep action requires only 'regex', and no other fields",
}, {
},
{
filename: "labelkeep3.bad.yml",
errMsg: "labelkeep action requires only 'regex', and no other fields",
}, {
},
{
filename: "labelkeep4.bad.yml",
errMsg: "labelkeep action requires only 'regex', and no other fields",
}, {
},
{
filename: "labelkeep5.bad.yml",
errMsg: "labelkeep action requires only 'regex', and no other fields",
}, {
},
{
filename: "labeldrop.bad.yml",
errMsg: "labeldrop action requires only 'regex', and no other fields",
}, {
},
{
filename: "labeldrop2.bad.yml",
errMsg: "labeldrop action requires only 'regex', and no other fields",
}, {
},
{
filename: "labeldrop3.bad.yml",
errMsg: "labeldrop action requires only 'regex', and no other fields",
}, {
},
{
filename: "labeldrop4.bad.yml",
errMsg: "labeldrop action requires only 'regex', and no other fields",
}, {
},
{
filename: "labeldrop5.bad.yml",
errMsg: "labeldrop action requires only 'regex', and no other fields",
}, {
},
{
filename: "labelmap.bad.yml",
errMsg: "\"l-$1\" is invalid 'replacement' for labelmap action",
}, {
},
{
filename: "rules.bad.yml",
errMsg: "invalid rule file path",
}, {
},
{
filename: "unknown_attr.bad.yml",
errMsg: "field consult_sd_configs not found in type",
}, {
},
{
filename: "bearertoken.bad.yml",
errMsg: "at most one of bearer_token & bearer_token_file must be configured",
}, {
},
{
filename: "bearertoken_basicauth.bad.yml",
errMsg: "at most one of basic_auth, oauth2, bearer_token & bearer_token_file must be configured",
}, {
},
{
filename: "kubernetes_http_config_without_api_server.bad.yml",
errMsg: "to use custom HTTP client configuration please provide the 'api_server' URL explicitly",
}, {
},
{
filename: "kubernetes_kubeconfig_with_apiserver.bad.yml",
errMsg: "cannot use 'kubeconfig_file' and 'api_server' simultaneously",
}, {
},
{
filename: "kubernetes_kubeconfig_with_http_config.bad.yml",
errMsg: "cannot use a custom HTTP client configuration together with 'kubeconfig_file'",
},
{
filename: "kubernetes_bearertoken.bad.yml",
errMsg: "at most one of bearer_token & bearer_token_file must be configured",
}, {
},
{
filename: "kubernetes_role.bad.yml",
errMsg: "role",
}, {
},
{
filename: "kubernetes_selectors_endpoints.bad.yml",
errMsg: "endpoints role supports only pod, service, endpoints selectors",
}, {
},
{
filename: "kubernetes_selectors_ingress.bad.yml",
errMsg: "ingress role supports only ingress selectors",
}, {
},
{
filename: "kubernetes_selectors_node.bad.yml",
errMsg: "node role supports only node selectors",
}, {
},
{
filename: "kubernetes_selectors_pod.bad.yml",
errMsg: "pod role supports only pod selectors",
}, {
},
{
filename: "kubernetes_selectors_service.bad.yml",
errMsg: "service role supports only service selectors",
}, {
},
{
filename: "kubernetes_namespace_discovery.bad.yml",
errMsg: "field foo not found in type kubernetes.plain",
}, {
},
{
filename: "kubernetes_selectors_duplicated_role.bad.yml",
errMsg: "duplicated selector role: pod",
}, {
},
{
filename: "kubernetes_selectors_incorrect_selector.bad.yml",
errMsg: "invalid selector: 'metadata.status-Running'; can't understand 'metadata.status-Running'",
}, {
},
{
filename: "kubernetes_bearertoken_basicauth.bad.yml",
errMsg: "at most one of basic_auth, oauth2, bearer_token & bearer_token_file must be configured",
}, {
},
{
filename: "kubernetes_authorization_basicauth.bad.yml",
errMsg: "at most one of basic_auth, oauth2 & authorization must be configured",
}, {
},
{
filename: "marathon_no_servers.bad.yml",
errMsg: "marathon_sd: must contain at least one Marathon server",
}, {
},
{
filename: "marathon_authtoken_authtokenfile.bad.yml",
errMsg: "marathon_sd: at most one of auth_token & auth_token_file must be configured",
}, {
},
{
filename: "marathon_authtoken_basicauth.bad.yml",
errMsg: "marathon_sd: at most one of basic_auth, auth_token & auth_token_file must be configured",
}, {
},
{
filename: "marathon_authtoken_bearertoken.bad.yml",
errMsg: "marathon_sd: at most one of bearer_token, bearer_token_file, auth_token & auth_token_file must be configured",
}, {
},
{
filename: "marathon_authtoken_authorization.bad.yml",
errMsg: "marathon_sd: at most one of auth_token, auth_token_file & authorization must be configured",
}, {
},
{
filename: "openstack_role.bad.yml",
errMsg: "unknown OpenStack SD role",
}, {
},
{
filename: "openstack_availability.bad.yml",
errMsg: "unknown availability invalid, must be one of admin, internal or public",
}, {
},
{
filename: "url_in_targetgroup.bad.yml",
errMsg: "\"http://bad\" is not a valid hostname",
}, {
},
{
filename: "target_label_missing.bad.yml",
errMsg: "relabel configuration for replace action requires 'target_label' value",
}, {
},
{
filename: "target_label_hashmod_missing.bad.yml",
errMsg: "relabel configuration for hashmod action requires 'target_label' value",
}, {
},
{
filename: "unknown_global_attr.bad.yml",
errMsg: "field nonexistent_field not found in type config.plain",
}, {
},
{
filename: "remote_read_url_missing.bad.yml",
errMsg: `url for remote_read is empty`,
}, {
},
{
filename: "remote_write_header.bad.yml",
errMsg: `x-prometheus-remote-write-version is a reserved header. It must not be changed`,
}, {
},
{
filename: "remote_read_header.bad.yml",
errMsg: `x-prometheus-remote-write-version is a reserved header. It must not be changed`,
}, {
},
{
filename: "remote_write_authorization_header.bad.yml",
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, or sigv4 parameter`,
}, {
},
{
filename: "remote_write_url_missing.bad.yml",
errMsg: `url for remote_write is empty`,
}, {
},
{
filename: "remote_write_dup.bad.yml",
errMsg: `found multiple remote write configs with job name "queue1"`,
}, {
},
{
filename: "remote_read_dup.bad.yml",
errMsg: `found multiple remote read configs with job name "queue1"`,
},
@ -1376,7 +1438,7 @@ var expectedErrors = []struct {
func TestBadConfigs(t *testing.T) {
for _, ee := range expectedErrors {
_, err := LoadFile("testdata/"+ee.filename, false, log.NewNopLogger())
_, err := LoadFile("testdata/"+ee.filename, false, false, log.NewNopLogger())
require.Error(t, err, "%s", ee.filename)
require.Contains(t, err.Error(), ee.errMsg,
"Expected error for %s to contain %q but got: %s", ee.filename, ee.errMsg, err)
@ -1410,20 +1472,20 @@ func TestExpandExternalLabels(t *testing.T) {
// Cleanup ant TEST env variable that could exist on the system.
os.Setenv("TEST", "")
c, err := LoadFile("testdata/external_labels.good.yml", false, log.NewNopLogger())
c, err := LoadFile("testdata/external_labels.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0])
require.Equal(t, labels.Label{Name: "baz", Value: "foo${TEST}bar"}, c.GlobalConfig.ExternalLabels[1])
require.Equal(t, labels.Label{Name: "foo", Value: "${TEST}"}, c.GlobalConfig.ExternalLabels[2])
c, err = LoadFile("testdata/external_labels.good.yml", true, log.NewNopLogger())
c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0])
require.Equal(t, labels.Label{Name: "baz", Value: "foobar"}, c.GlobalConfig.ExternalLabels[1])
require.Equal(t, labels.Label{Name: "foo", Value: ""}, c.GlobalConfig.ExternalLabels[2])
os.Setenv("TEST", "TestValue")
c, err = LoadFile("testdata/external_labels.good.yml", true, log.NewNopLogger())
c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0])
require.Equal(t, labels.Label{Name: "baz", Value: "fooTestValuebar"}, c.GlobalConfig.ExternalLabels[1])

3
config/testdata/conf.good.yml vendored
View file

@ -23,6 +23,9 @@ remote_write:
client_id: "123"
client_secret: "456"
token_url: "http://remote1/auth"
tls_config:
cert_file: valid_cert_file
key_file: valid_key_file
- url: http://remote2/push
name: rw_tls

2
discovery/README.md
View file

@ -131,7 +131,7 @@ the Prometheus server will be able to see them.
### The SD interface
A Service Discovery (SD) mechanism has to discover targets and provide them to Prometheus. We expect similar targets to be grouped together, in the form of a [target group](https://pkg.go.dev/github.com/prometheus/prometheus/discovery/targetgroup#Group). The SD mechanism sends the targets down to prometheus as list of target groups.
A Service Discovery (SD) mechanism has to discover targets and provide them to Prometheus. We expect similar targets to be grouped together, in the form of a [target group](https://pkg.go.dev/github.com/prometheus/prometheus@v1.8.2-0.20211105201321-411021ada9ab/discovery/targetgroup#Group). The SD mechanism sends the targets down to prometheus as list of target groups.
An SD mechanism has to implement the `Discoverer` Interface:
```go

12
discovery/aws/ec2.go
View file

@ -63,13 +63,11 @@ const (
ec2LabelSeparator = ","
)
var (
// DefaultEC2SDConfig is the default EC2 SD configuration.
DefaultEC2SDConfig = EC2SDConfig{
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
}
)
// DefaultEC2SDConfig is the default EC2 SD configuration.
var DefaultEC2SDConfig = EC2SDConfig{
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
}
func init() {
discovery.RegisterConfig(&EC2SDConfig{})

12
discovery/aws/lightsail.go
View file

@ -53,13 +53,11 @@ const (
lightsailLabelSeparator = ","
)
var (
// DefaultLightsailSDConfig is the default Lightsail SD configuration.
DefaultLightsailSDConfig = LightsailSDConfig{
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
}
)
// DefaultLightsailSDConfig is the default Lightsail SD configuration.
var DefaultLightsailSDConfig = LightsailSDConfig{
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
}
func init() {
discovery.RegisterConfig(&LightsailSDConfig{})

17
discovery/azure/azure.go
View file

@ -64,6 +64,7 @@ var DefaultSDConfig = SDConfig{
RefreshInterval: model.Duration(5 * time.Minute),
Environment: azure.PublicCloud.Name,
AuthenticationMethod: authMethodOAuth,
HTTPClientConfig: config_util.DefaultHTTPClientConfig,
}
func init() {
@ -80,6 +81,8 @@ type SDConfig struct {
ClientSecret config_util.Secret `yaml:"client_secret,omitempty"`
RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
AuthenticationMethod string `yaml:"authentication_method,omitempty"`
HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
}
// Name returns the name of the Config.
@ -200,19 +203,29 @@ func createAzureClient(cfg SDConfig) (azureClient, error) {
}
}
client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "azure_sd")
if err != nil {
return azureClient{}, err
}
sender := autorest.DecorateSender(client)
bearerAuthorizer := autorest.NewBearerAuthorizer(spt)
c.vm = compute.NewVirtualMachinesClientWithBaseURI(resourceManagerEndpoint, cfg.SubscriptionID)
c.vm.Authorizer = bearerAuthorizer
c.vm.Sender = sender
c.nic = network.NewInterfacesClientWithBaseURI(resourceManagerEndpoint, cfg.SubscriptionID)
c.nic.Authorizer = bearerAuthorizer
c.nic.Sender = sender
c.vmss = compute.NewVirtualMachineScaleSetsClientWithBaseURI(resourceManagerEndpoint, cfg.SubscriptionID)
c.vmss.Authorizer = bearerAuthorizer
c.vm.Sender = sender
c.vmssvm = compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(resourceManagerEndpoint, cfg.SubscriptionID)
c.vmssvm.Authorizer = bearerAuthorizer
c.vmssvm.Sender = sender
return c, nil
}
@ -326,7 +339,6 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
// Get the IP address information via separate call to the network provider.
for _, nicID := range vm.NetworkInterfaces {
networkInterface, err := client.getNetworkInterfaceByID(ctx, nicID)
if err != nil {
level.Error(d.logger).Log("msg", "Unable to get network interface", "name", nicID, "err", err)
ch <- target{labelSet: nil, err: err}
@ -424,9 +436,8 @@ func (client *azureClient) getScaleSets(ctx context.Context) ([]compute.VirtualM
func (client *azureClient) getScaleSetVMs(ctx context.Context, scaleSet compute.VirtualMachineScaleSet) ([]virtualMachine, error) {
var vms []virtualMachine
//TODO do we really need to fetch the resourcegroup this way?
// TODO do we really need to fetch the resourcegroup this way?
r, err := newAzureResourceFromID(*scaleSet.ID, nil)
if err != nil {
return nil, errors.Wrap(err, "could not parse scale set ID")
}

5
discovery/consul/consul.go
View file

@ -54,7 +54,7 @@ const (
healthLabel = model.MetaLabelPrefix + "consul_health"
// serviceAddressLabel is the name of the label containing the (optional) service address.
serviceAddressLabel = model.MetaLabelPrefix + "consul_service_address"
//servicePortLabel is the name of the label containing the service port.
// servicePortLabel is the name of the label containing the service port.
servicePortLabel = model.MetaLabelPrefix + "consul_service_port"
// datacenterLabel is the name of the label containing the datacenter ID.
datacenterLabel = model.MetaLabelPrefix + "consul_dc"
@ -297,6 +297,7 @@ func (d *Discovery) getDatacenter() error {
}
d.clientDatacenter = dc
d.logger = log.With(d.logger, "datacenter", dc)
return nil
}
@ -530,7 +531,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
for _, serviceNode := range serviceNodes {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
var tags = srv.tagSeparator + strings.Join(serviceNode.Service.Tags, srv.tagSeparator) + srv.tagSeparator
tags := srv.tagSeparator + strings.Join(serviceNode.Service.Tags, srv.tagSeparator) + srv.tagSeparator
// If the service address is not empty it should be used instead of the node address
// since the service may be registered remotely through a different node.

13
discovery/consul/consul_test.go
View file

@ -37,9 +37,9 @@ func TestMain(m *testing.M) {
func TestConfiguredService(t *testing.T) {
conf := &SDConfig{
Services: []string{"configuredServiceName"}}
Services: []string{"configuredServiceName"},
}
consulDiscovery, err := NewDiscovery(conf, nil)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
@ -57,7 +57,6 @@ func TestConfiguredServiceWithTag(t *testing.T) {
ServiceTags: []string{"http"},
}
consulDiscovery, err := NewDiscovery(conf, nil)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
@ -153,7 +152,6 @@ func TestConfiguredServiceWithTags(t *testing.T) {
for _, tc := range cases {
consulDiscovery, err := NewDiscovery(tc.conf, nil)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
@ -168,7 +166,6 @@ func TestConfiguredServiceWithTags(t *testing.T) {
func TestNonConfiguredService(t *testing.T) {
conf := &SDConfig{}
consulDiscovery, err := NewDiscovery(conf, nil)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
@ -310,11 +307,15 @@ func TestNoTargets(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan []*targetgroup.Group)
go d.Run(ctx, ch)
go func() {
d.Run(ctx, ch)
close(ch)
}()
targets := (<-ch)[0].Targets
require.Equal(t, 0, len(targets))
cancel()
<-ch
}
// Watch only the test service.

3
discovery/digitalocean/mock_test.go
View file

@ -75,7 +75,8 @@ func (m *SDMock) HandleDropletsList() {
panic(err)
}
}
fmt.Fprint(w, []string{`
fmt.Fprint(w, []string{
`
{
"droplets": [
{

2
discovery/file/file.go
View file

@ -25,13 +25,13 @@ import (
"sync"
"time"
"github.com/fsnotify/fsnotify"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
fsnotify "gopkg.in/fsnotify/fsnotify.v1"
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/discovery"

7
discovery/file/file_test.go
View file

@ -73,7 +73,7 @@ func (t *testRunner) copyFile(src string) string {
}
// copyFileTo atomically copies a file with a different name to the runner's directory.
func (t *testRunner) copyFileTo(src string, name string) string {
func (t *testRunner) copyFileTo(src, name string) string {
t.Helper()
newf, err := ioutil.TempFile(t.dir, "")
@ -95,7 +95,7 @@ func (t *testRunner) copyFileTo(src string, name string) string {
}
// writeString writes atomically a string to a file.
func (t *testRunner) writeString(file string, data string) {
func (t *testRunner) writeString(file, data string) {
t.Helper()
newf, err := ioutil.TempFile(t.dir, "")
@ -477,6 +477,7 @@ func TestRemoveFile(t *testing.T) {
},
{
Source: fileSource(sdFile, 1),
}},
},
},
)
}

1
discovery/hetzner/hcloud.go
View file

@ -78,6 +78,7 @@ func newHcloudDiscovery(conf *SDConfig, logger log.Logger) (*hcloudDiscovery, er
)
return d, nil
}
func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
servers, err := d.client.Server.All(ctx)
if err != nil {

6
discovery/hetzner/mock_test.go
View file

@ -489,8 +489,10 @@ func (m *SDMock) HandleHcloudNetworks() {
})
}
const robotTestUsername = "my-hetzner"
const robotTestPassword = "my-password"
const (
robotTestUsername = "my-hetzner"
robotTestPassword = "my-password"
)
// HandleRobotServers mocks the robot servers list endpoint.
func (m *SDMock) HandleRobotServers() {

1
discovery/hetzner/robot.go
View file

@ -70,6 +70,7 @@ func newRobotDiscovery(conf *SDConfig, logger log.Logger) (*robotDiscovery, erro
return d, nil
}
func (d *robotDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
req, err := http.NewRequest("GET", d.endpoint+"/server", nil)
if err != nil {

5
discovery/http/http_test.go
View file

@ -24,8 +24,9 @@ import (
"github.com/go-kit/log"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func TestHTTPValidRefresh(t *testing.T) {
@ -60,7 +61,6 @@ func TestHTTPValidRefresh(t *testing.T) {
},
}
require.Equal(t, tgs, expectedTargets)
}
func TestHTTPInvalidCode(t *testing.T) {
@ -398,5 +398,4 @@ func TestSourceDisappeared(t *testing.T) {
require.Equal(t, test.expectedTargets[i], tgs)
}
}
}

10
discovery/kubernetes/client_metrics.go
View file

@ -121,9 +121,11 @@ func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer
clientGoRequestLatencyMetricVec,
)
}
func (clientGoRequestMetricAdapter) Increment(ctx context.Context, code string, method string, host string) {
func (clientGoRequestMetricAdapter) Increment(ctx context.Context, code, method, host string) {
clientGoRequestResultMetricVec.WithLabelValues(code).Inc()
}
func (clientGoRequestMetricAdapter) Observe(ctx context.Context, verb string, u url.URL, latency time.Duration) {
clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
}
@ -146,21 +148,27 @@ func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Regist
func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
return clientGoWorkqueueDepthMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
return clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
return clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
return clientGoWorkqueueUnfinishedWorkSecondsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric {
return clientGoWorkqueueLongestRunningProcessorMetricVec.WithLabelValues(name)
}
func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
// Retries are not used so the metric is omitted.
return noopMetric{}

3
discovery/kubernetes/endpoints.go
View file

@ -18,8 +18,6 @@ import (
"net"
"strconv"
"github.com/prometheus/prometheus/util/strutil"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/pkg/errors"
@ -29,6 +27,7 @@ import (
"k8s.io/client-go/util/workqueue"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
var (

2
discovery/kubernetes/endpoints_test.go
View file

@ -27,7 +27,7 @@ import (
)
func makeEndpoints() *v1.Endpoints {
var nodeName = "foobar"
nodeName := "foobar"
return &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",

25
discovery/kubernetes/kubernetes_test.go
View file

@ -86,15 +86,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
// Ensure that discovery has a discoverer set. This prevents a race
// condition where the above go routine may or may not have set a
// discoverer yet.
lastDiscoverersCount := 0
dis := d.discovery.(*Discovery)
for {
dis := d.discovery.(*Discovery)
dis.RLock()
l := len(dis.discoverers)
dis.RUnlock()
if l > 0 {
if l > 0 && l == lastDiscoverersCount {
break
}
time.Sleep(10 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
lastDiscoverersCount = l
}
resChan := make(chan map[string]*targetgroup.Group)
@ -171,13 +174,15 @@ type hasSynced interface {
hasSynced() bool
}
var _ hasSynced = &Discovery{}
var _ hasSynced = &Node{}
var _ hasSynced = &Endpoints{}
var _ hasSynced = &EndpointSlice{}
var _ hasSynced = &Ingress{}
var _ hasSynced = &Pod{}
var _ hasSynced = &Service{}
var (
_ hasSynced = &Discovery{}
_ hasSynced = &Node{}
_ hasSynced = &Endpoints{}
_ hasSynced = &EndpointSlice{}
_ hasSynced = &Ingress{}
_ hasSynced = &Pod{}
_ hasSynced = &Service{}
)
func (d *Discovery) hasSynced() bool {
d.RLock()

2
discovery/kubernetes/node_test.go
View file

@ -25,7 +25,7 @@ import (
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func makeNode(name, address string, labels map[string]string, annotations map[string]string) *v1.Node {
func makeNode(name, address string, labels, annotations map[string]string) *v1.Node {
return &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: name,

357
discovery/legacymanager/manager.go Normal file
View file

@ -0,0 +1,357 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package legacymanager
import (
"context"
"fmt"
"reflect"
"sync"
"time"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
var (
failedConfigs = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "prometheus_sd_failed_configs",
Help: "Current number of service discovery configurations that failed to load.",
},
[]string{"name"},
)
discoveredTargets = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "prometheus_sd_discovered_targets",
Help: "Current number of discovered targets.",
},
[]string{"name", "config"},
)
receivedUpdates = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_sd_received_updates_total",
Help: "Total number of update events received from the SD providers.",
},
[]string{"name"},
)
delayedUpdates = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_sd_updates_delayed_total",
Help: "Total number of update events that couldn't be sent immediately.",
},
[]string{"name"},
)
sentUpdates = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_sd_updates_total",
Help: "Total number of update events sent to the SD consumers.",
},
[]string{"name"},
)
)
func RegisterMetrics() {
prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
}
type poolKey struct {
setName string
provider string
}
// provider holds a Discoverer instance, its configuration and its subscribers.
type provider struct {
name string
d discovery.Discoverer
subs []string
config interface{}
}
// NewManager is the Discovery Manager constructor.
func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
if logger == nil {
logger = log.NewNopLogger()
}
mgr := &Manager{
logger: logger,
syncCh: make(chan map[string][]*targetgroup.Group),
targets: make(map[poolKey]map[string]*targetgroup.Group),
discoverCancel: []context.CancelFunc{},
ctx: ctx,
updatert: 5 * time.Second,
triggerSend: make(chan struct{}, 1),
}
for _, option := range options {
option(mgr)
}
return mgr
}
// Name sets the name of the manager.
func Name(n string) func(*Manager) {
return func(m *Manager) {
m.mtx.Lock()
defer m.mtx.Unlock()
m.name = n
}
}
// Manager maintains a set of discovery providers and sends each update to a map channel.
// Targets are grouped by the target set name.
type Manager struct {
logger log.Logger
name string
mtx sync.RWMutex
ctx context.Context
discoverCancel []context.CancelFunc
// Some Discoverers(eg. k8s) send only the updates for a given target group
// so we use map[tg.Source]*targetgroup.Group to know which group to update.
targets map[poolKey]map[string]*targetgroup.Group
// providers keeps track of SD providers.
providers []*provider
// The sync channel sends the updates as a map where the key is the job value from the scrape config.
syncCh chan map[string][]*targetgroup.Group
// How long to wait before sending updates to the channel. The variable
// should only be modified in unit tests.
updatert time.Duration
// The triggerSend channel signals to the manager that new updates have been received from providers.
triggerSend chan struct{}
}
// Run starts the background processing
func (m *Manager) Run() error {
go m.sender()
for range m.ctx.Done() {
m.cancelDiscoverers()
return m.ctx.Err()
}
return nil
}
// SyncCh returns a read only channel used by all the clients to receive target updates.
func (m *Manager) SyncCh() <-chan map[string][]*targetgroup.Group {
return m.syncCh
}
// ApplyConfig removes all running discovery providers and starts new ones using the provided config.
func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
m.mtx.Lock()
defer m.mtx.Unlock()
for pk := range m.targets {
if _, ok := cfg[pk.setName]; !ok {
discoveredTargets.DeleteLabelValues(m.name, pk.setName)
}
}
m.cancelDiscoverers()
m.targets = make(map[poolKey]map[string]*targetgroup.Group)
m.providers = nil
m.discoverCancel = nil
failedCount := 0
for name, scfg := range cfg {
failedCount += m.registerProviders(scfg, name)
discoveredTargets.WithLabelValues(m.name, name).Set(0)
}
failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
for _, prov := range m.providers {
m.startProvider(m.ctx, prov)
}
return nil
}
// StartCustomProvider is used for sdtool. Only use this if you know what you're doing.
func (m *Manager) StartCustomProvider(ctx context.Context, name string, worker discovery.Discoverer) {
p := &provider{
name: name,
d: worker,
subs: []string{name},
}
m.providers = append(m.providers, p)
m.startProvider(ctx, p)
}
func (m *Manager) startProvider(ctx context.Context, p *provider) {
level.Debug(m.logger).Log("msg", "Starting provider", "provider", p.name, "subs", fmt.Sprintf("%v", p.subs))
ctx, cancel := context.WithCancel(ctx)
updates := make(chan []*targetgroup.Group)
m.discoverCancel = append(m.discoverCancel, cancel)
go p.d.Run(ctx, updates)
go m.updater(ctx, p, updates)
}
func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targetgroup.Group) {
for {
select {
case <-ctx.Done():
return
case tgs, ok := <-updates:
receivedUpdates.WithLabelValues(m.name).Inc()
if !ok {
level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
return
}
for _, s := range p.subs {
m.updateGroup(poolKey{setName: s, provider: p.name}, tgs)
}
select {
case m.triggerSend <- struct{}{}:
default:
}
}
}
}
func (m *Manager) sender() {
ticker := time.NewTicker(m.updatert)
defer ticker.Stop()
for {
select {
case <-m.ctx.Done():
return
case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
select {
case <-m.triggerSend:
sentUpdates.WithLabelValues(m.name).Inc()
select {
case m.syncCh <- m.allGroups():
default:
delayedUpdates.WithLabelValues(m.name).Inc()
level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
select {
case m.triggerSend <- struct{}{}:
default:
}
}
default:
}
}
}
}
func (m *Manager) cancelDiscoverers() {
for _, c := range m.discoverCancel {
c()
}
}
func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) {
m.mtx.Lock()
defer m.mtx.Unlock()
if _, ok := m.targets[poolKey]; !ok {
m.targets[poolKey] = make(map[string]*targetgroup.Group)
}
for _, tg := range tgs {
if tg != nil { // Some Discoverers send nil target group so need to check for it to avoid panics.
m.targets[poolKey][tg.Source] = tg
}
}
}
func (m *Manager) allGroups() map[string][]*targetgroup.Group {
m.mtx.RLock()
defer m.mtx.RUnlock()
tSets := map[string][]*targetgroup.Group{}
n := map[string]int{}
for pkey, tsets := range m.targets {
for _, tg := range tsets {
// Even if the target group 'tg' is empty we still need to send it to the 'Scrape manager'
// to signal that it needs to stop all scrape loops for this target set.
tSets[pkey.setName] = append(tSets[pkey.setName], tg)
n[pkey.setName] += len(tg.Targets)
}
}
for setName, v := range n {
discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
}
return tSets
}
// registerProviders returns a number of failed SD config.
func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int {
var (
failed int
added bool
)
add := func(cfg discovery.Config) {
for _, p := range m.providers {
if reflect.DeepEqual(cfg, p.config) {
p.subs = append(p.subs, setName)
added = true
return
}
}
typ := cfg.Name()
d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
Logger: log.With(m.logger, "discovery", typ),
})
if err != nil {
level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ)
failed++
return
}
m.providers = append(m.providers, &provider{
name: fmt.Sprintf("%s/%d", typ, len(m.providers)),
d: d,
config: cfg,
subs: []string{setName},
})
added = true
}
for _, cfg := range cfgs {
add(cfg)
}
if !added {
// Add an empty target group to force the refresh of the corresponding
// scrape pool and to notify the receiver that this target set has no
// current targets.
// It can happen because the combined set of SD configurations is empty
// or because we fail to instantiate all the SD configurations.
add(discovery.StaticConfig{{}})
}
return failed
}
// StaticProvider holds a list of target groups that never change.
type StaticProvider struct {
TargetGroups []*targetgroup.Group
}
// Run implements the Worker interface.
func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
// We still have to consider that the consumer exits right away in which case
// the context will be canceled.
select {
case ch <- sd.TargetGroups:
case <-ctx.Done():
}
close(ch)
}

1140
discovery/legacymanager/manager_test.go Normal file
View file

File diff suppressed because it is too large Load diff

259
discovery/legacymanager/registry.go Normal file
View file

@ -0,0 +1,259 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package legacymanager
import (
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
const (
configFieldPrefix = "AUTO_DISCOVERY_"
staticConfigsKey = "static_configs"
staticConfigsFieldName = configFieldPrefix + staticConfigsKey
)
var (
configNames = make(map[string]discovery.Config)
configFieldNames = make(map[reflect.Type]string)
configFields []reflect.StructField
configTypesMu sync.Mutex
configTypes = make(map[reflect.Type]reflect.Type)
emptyStructType = reflect.TypeOf(struct{}{})
configsType = reflect.TypeOf(discovery.Configs{})
)
// RegisterConfig registers the given Config type for YAML marshaling and unmarshaling.
func RegisterConfig(config discovery.Config) {
registerConfig(config.Name()+"_sd_configs", reflect.TypeOf(config), config)
}
func init() {
// N.B.: static_configs is the only Config type implemented by default.
// All other types are registered at init by their implementing packages.
elemTyp := reflect.TypeOf(&targetgroup.Group{})
registerConfig(staticConfigsKey, elemTyp, discovery.StaticConfig{})
}
func registerConfig(yamlKey string, elemType reflect.Type, config discovery.Config) {
name := config.Name()
if _, ok := configNames[name]; ok {
panic(fmt.Sprintf("discovery: Config named %q is already registered", name))
}
configNames[name] = config
fieldName := configFieldPrefix + yamlKey // Field must be exported.
configFieldNames[elemType] = fieldName
// Insert fields in sorted order.
i := sort.Search(len(configFields), func(k int) bool {
return fieldName < configFields[k].Name
})
configFields = append(configFields, reflect.StructField{}) // Add empty field at end.
copy(configFields[i+1:], configFields[i:]) // Shift fields to the right.
configFields[i] = reflect.StructField{ // Write new field in place.
Name: fieldName,
Type: reflect.SliceOf(elemType),
Tag: reflect.StructTag(`yaml:"` + yamlKey + `,omitempty"`),
}
}
func getConfigType(out reflect.Type) reflect.Type {
configTypesMu.Lock()
defer configTypesMu.Unlock()
if typ, ok := configTypes[out]; ok {
return typ
}
// Initial exported fields map one-to-one.
var fields []reflect.StructField
for i, n := 0, out.NumField(); i < n; i++ {
switch field := out.Field(i); {
case field.PkgPath == "" && field.Type != configsType:
fields = append(fields, field)
default:
fields = append(fields, reflect.StructField{
Name: "_" + field.Name, // Field must be unexported.
PkgPath: out.PkgPath(),
Type: emptyStructType,
})
}
}
// Append extra config fields on the end.
fields = append(fields, configFields...)
typ := reflect.StructOf(fields)
configTypes[out] = typ
return typ
}
// UnmarshalYAMLWithInlineConfigs helps implement yaml.Unmarshal for structs
// that have a Configs field that should be inlined.
func UnmarshalYAMLWithInlineConfigs(out interface{}, unmarshal func(interface{}) error) error {
outVal := reflect.ValueOf(out)
if outVal.Kind() != reflect.Ptr {
return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out)
}
outVal = outVal.Elem()
if outVal.Kind() != reflect.Struct {
return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out)
}
outTyp := outVal.Type()
cfgTyp := getConfigType(outTyp)
cfgPtr := reflect.New(cfgTyp)
cfgVal := cfgPtr.Elem()
// Copy shared fields (defaults) to dynamic value.
var configs *discovery.Configs
for i, n := 0, outVal.NumField(); i < n; i++ {
if outTyp.Field(i).Type == configsType {
configs = outVal.Field(i).Addr().Interface().(*discovery.Configs)
continue
}
if cfgTyp.Field(i).PkgPath != "" {
continue // Field is unexported: ignore.
}
cfgVal.Field(i).Set(outVal.Field(i))
}
if configs == nil {
return fmt.Errorf("discovery: Configs field not found in type: %T", out)
}
// Unmarshal into dynamic value.
if err := unmarshal(cfgPtr.Interface()); err != nil {
return replaceYAMLTypeError(err, cfgTyp, outTyp)
}
// Copy shared fields from dynamic value.
for i, n := 0, outVal.NumField(); i < n; i++ {
if cfgTyp.Field(i).PkgPath != "" {
continue // Field is unexported: ignore.
}
outVal.Field(i).Set(cfgVal.Field(i))
}
var err error
*configs, err = readConfigs(cfgVal, outVal.NumField())
return err
}
func readConfigs(structVal reflect.Value, startField int) (discovery.Configs, error) {
var (
configs discovery.Configs
targets []*targetgroup.Group
)
for i, n := startField, structVal.NumField(); i < n; i++ {
field := structVal.Field(i)
if field.Kind() != reflect.Slice {
panic("discovery: internal error: field is not a slice")
}
for k := 0; k < field.Len(); k++ {
val := field.Index(k)
if val.IsZero() || (val.Kind() == reflect.Ptr && val.Elem().IsZero()) {
key := configFieldNames[field.Type().Elem()]
key = strings.TrimPrefix(key, configFieldPrefix)
return nil, fmt.Errorf("empty or null section in %s", key)
}
switch c := val.Interface().(type) {
case *targetgroup.Group:
// Add index to the static config target groups for unique identification
// within scrape pool.
c.Source = strconv.Itoa(len(targets))
// Coalesce multiple static configs into a single static config.
targets = append(targets, c)
case discovery.Config:
configs = append(configs, c)
default:
panic("discovery: internal error: slice element is not a Config")
}
}
}
if len(targets) > 0 {
configs = append(configs, discovery.StaticConfig(targets))
}
return configs, nil
}
// MarshalYAMLWithInlineConfigs helps implement yaml.Marshal for structs
// that have a Configs field that should be inlined.
func MarshalYAMLWithInlineConfigs(in interface{}) (interface{}, error) {
inVal := reflect.ValueOf(in)
for inVal.Kind() == reflect.Ptr {
inVal = inVal.Elem()
}
inTyp := inVal.Type()
cfgTyp := getConfigType(inTyp)
cfgPtr := reflect.New(cfgTyp)
cfgVal := cfgPtr.Elem()
// Copy shared fields to dynamic value.
var configs *discovery.Configs
for i, n := 0, inTyp.NumField(); i < n; i++ {
if inTyp.Field(i).Type == configsType {
configs = inVal.Field(i).Addr().Interface().(*discovery.Configs)
}
if cfgTyp.Field(i).PkgPath != "" {
continue // Field is unexported: ignore.
}
cfgVal.Field(i).Set(inVal.Field(i))
}
if configs == nil {
return nil, fmt.Errorf("discovery: Configs field not found in type: %T", in)
}
if err := writeConfigs(cfgVal, *configs); err != nil {
return nil, err
}
return cfgPtr.Interface(), nil
}
func writeConfigs(structVal reflect.Value, configs discovery.Configs) error {
targets := structVal.FieldByName(staticConfigsFieldName).Addr().Interface().(*[]*targetgroup.Group)
for _, c := range configs {
if sc, ok := c.(discovery.StaticConfig); ok {
*targets = append(*targets, sc...)
continue
}
fieldName, ok := configFieldNames[reflect.TypeOf(c)]
if !ok {
return fmt.Errorf("discovery: cannot marshal unregistered Config type: %T", c)
}
field := structVal.FieldByName(fieldName)
field.Set(reflect.Append(field, reflect.ValueOf(c)))
}
return nil
}
func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error {
if e, ok := err.(*yaml.TypeError); ok {
oldStr := oldTyp.String()
newStr := newTyp.String()
for i, s := range e.Errors {
e.Errors[i] = strings.Replace(s, oldStr, newStr, -1)
}
}
return err
}

12
discovery/linode/linode.go
View file

@ -161,8 +161,12 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
if d.lastResults != nil && d.eventPollingEnabled {
// Check to see if there have been any events. If so, refresh our data.
opts := linodego.NewListOptions(1, fmt.Sprintf(filterTemplate, d.lastRefreshTimestamp.Format("2006-01-02T15:04:05")))
events, err := d.client.ListEvents(ctx, opts)
opts := linodego.ListOptions{
PageOptions: &linodego.PageOptions{Page: 1},
PageSize: 25,
Filter: fmt.Sprintf(filterTemplate, d.lastRefreshTimestamp.Format("2006-01-02T15:04:05")),
}
events, err := d.client.ListEvents(ctx, &opts)
if err != nil {
var e *linodego.Error
if errors.As(err, &e) && e.Code == http.StatusUnauthorized {
@ -205,13 +209,13 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
}
// Gather all linode instances.
instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{})
instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
if err != nil {
return nil, err
}
// Gather detailed IP address info for all IPs on all linode instances.
detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{})
detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
if err != nil {
return nil, err
}

2
discovery/linode/linode_test.go
View file

@ -56,7 +56,7 @@ func TestLinodeSDRefresh(t *testing.T) {
require.NoError(t, err)
endpoint, err := url.Parse(sdmock.Mock.Endpoint())
require.NoError(t, err)
d.client.SetBaseURL(fmt.Sprintf("%s/v4", endpoint.String()))
d.client.SetBaseURL(endpoint.String())
tgs, err := d.refresh(context.Background())
require.NoError(t, err)

191
discovery/manager.go
View file

@ -65,7 +65,7 @@ var (
)
)
func init() {
func RegisterMetrics() {
prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
}
@ -74,12 +74,26 @@ type poolKey struct {
provider string
}
// provider holds a Discoverer instance, its configuration and its subscribers.
// provider holds a Discoverer instance, its configuration, cancel func and its subscribers.
type provider struct {
name string
d Discoverer
subs []string
config interface{}
cancel context.CancelFunc
// done should be called after cleaning up resources associated with cancelled provider.
done func()
mu sync.RWMutex
subs map[string]struct{}
// newSubs is used to temporary store subs to be used upon config reload completion.
newSubs map[string]struct{}
}
// IsStarted return true if Discoverer is started.
func (p *provider) IsStarted() bool {
return p.cancel != nil
}
// NewManager is the Discovery Manager constructor.
@ -88,13 +102,12 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
logger = log.NewNopLogger()
}
mgr := &Manager{
logger: logger,
syncCh: make(chan map[string][]*targetgroup.Group),
targets: make(map[poolKey]map[string]*targetgroup.Group),
discoverCancel: []context.CancelFunc{},
ctx: ctx,
updatert: 5 * time.Second,
triggerSend: make(chan struct{}, 1),
logger: logger,
syncCh: make(chan map[string][]*targetgroup.Group),
targets: make(map[poolKey]map[string]*targetgroup.Group),
ctx: ctx,
updatert: 5 * time.Second,
triggerSend: make(chan struct{}, 1),
}
for _, option := range options {
option(mgr)
@ -114,15 +127,16 @@ func Name(n string) func(*Manager) {
// Manager maintains a set of discovery providers and sends each update to a map channel.
// Targets are grouped by the target set name.
type Manager struct {
logger log.Logger
name string
mtx sync.RWMutex
ctx context.Context
discoverCancel []context.CancelFunc
logger log.Logger
name string
mtx sync.RWMutex
ctx context.Context
// Some Discoverers(eg. k8s) send only the updates for a given target group
// Some Discoverers(e.g. k8s) send only the updates for a given target group,
// so we use map[tg.Source]*targetgroup.Group to know which group to update.
targets map[poolKey]map[string]*targetgroup.Group
targets map[poolKey]map[string]*targetgroup.Group
targetsMtx sync.Mutex
// providers keeps track of SD providers.
providers []*provider
// The sync channel sends the updates as a map where the key is the job value from the scrape config.
@ -132,11 +146,14 @@ type Manager struct {
// should only be modified in unit tests.
updatert time.Duration
// The triggerSend channel signals to the manager that new updates have been received from providers.
// The triggerSend channel signals to the Manager that new updates have been received from providers.
triggerSend chan struct{}
// lastProvider counts providers registered during Manager's lifetime.
lastProvider uint
}
// Run starts the background processing
// Run starts the background processing.
func (m *Manager) Run() error {
go m.sender()
for range m.ctx.Done() {
@ -151,31 +168,82 @@ func (m *Manager) SyncCh() <-chan map[string][]*targetgroup.Group {
return m.syncCh
}
// ApplyConfig removes all running discovery providers and starts new ones using the provided config.
// ApplyConfig checks if discovery provider with supplied config is already running and keeps them as is.
// Remaining providers are then stopped and new required providers are started using the provided config.
func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
m.mtx.Lock()
defer m.mtx.Unlock()
for pk := range m.targets {
if _, ok := cfg[pk.setName]; !ok {
discoveredTargets.DeleteLabelValues(m.name, pk.setName)
}
}
m.cancelDiscoverers()
m.targets = make(map[poolKey]map[string]*targetgroup.Group)
m.providers = nil
m.discoverCancel = nil
failedCount := 0
var failedCount int
for name, scfg := range cfg {
failedCount += m.registerProviders(scfg, name)
discoveredTargets.WithLabelValues(m.name, name).Set(0)
}
failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
var (
wg sync.WaitGroup
// keep shows if we keep any providers after reload.
keep bool
newProviders []*provider
)
for _, prov := range m.providers {
m.startProvider(m.ctx, prov)
// Cancel obsolete providers.
if len(prov.newSubs) == 0 {
wg.Add(1)
prov.done = func() {
wg.Done()
}
prov.cancel()
continue
}
newProviders = append(newProviders, prov)
// refTargets keeps reference targets used to populate new subs' targets
var refTargets map[string]*targetgroup.Group
prov.mu.Lock()
m.targetsMtx.Lock()
for s := range prov.subs {
keep = true
refTargets = m.targets[poolKey{s, prov.name}]
// Remove obsolete subs' targets.
if _, ok := prov.newSubs[s]; !ok {
delete(m.targets, poolKey{s, prov.name})
discoveredTargets.DeleteLabelValues(m.name, s)
}
}
// Set metrics and targets for new subs.
for s := range prov.newSubs {
if _, ok := prov.subs[s]; !ok {
discoveredTargets.WithLabelValues(m.name, s).Set(0)
}
if l := len(refTargets); l > 0 {
m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l)
for k, v := range refTargets {
m.targets[poolKey{s, prov.name}][k] = v
}
}
}
m.targetsMtx.Unlock()
prov.subs = prov.newSubs
prov.newSubs = map[string]struct{}{}
prov.mu.Unlock()
if !prov.IsStarted() {
m.startProvider(m.ctx, prov)
}
}
// Currently downstream managers expect full target state upon config reload, so we must oblige.
// While startProvider does pull the trigger, it may take some time to do so, therefore
// we pull the trigger as soon as possible so that downstream managers can populate their state.
// See https://github.com/prometheus/prometheus/pull/8639 for details.
if keep {
select {
case m.triggerSend <- struct{}{}:
default:
}
}
m.providers = newProviders
wg.Wait()
return nil
}
@ -185,7 +253,9 @@ func (m *Manager) StartCustomProvider(ctx context.Context, name string, worker D
p := &provider{
name: name,
d: worker,
subs: []string{name},
subs: map[string]struct{}{
name: {},
},
}
m.providers = append(m.providers, p)
m.startProvider(ctx, p)
@ -196,13 +266,29 @@ func (m *Manager) startProvider(ctx context.Context, p *provider) {
ctx, cancel := context.WithCancel(ctx)
updates := make(chan []*targetgroup.Group)
m.discoverCancel = append(m.discoverCancel, cancel)
p.cancel = cancel
go p.d.Run(ctx, updates)
go m.updater(ctx, p, updates)
}
// cleaner cleans resources associated with provider.
func (m *Manager) cleaner(p *provider) {
m.targetsMtx.Lock()
p.mu.RLock()
for s := range p.subs {
delete(m.targets, poolKey{s, p.name})
}
p.mu.RUnlock()
m.targetsMtx.Unlock()
if p.done != nil {
p.done()
}
}
func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targetgroup.Group) {
// Ensure targets from this provider are cleaned up.
defer m.cleaner(p)
for {
select {
case <-ctx.Done():
@ -211,12 +297,16 @@ func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targ
receivedUpdates.WithLabelValues(m.name).Inc()
if !ok {
level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
// Wait for provider cancellation to ensure targets are cleaned up when expected.
<-ctx.Done()
return
}
for _, s := range p.subs {
p.mu.RLock()
for s := range p.subs {
m.updateGroup(poolKey{setName: s, provider: p.name}, tgs)
}
p.mu.RUnlock()
select {
case m.triggerSend <- struct{}{}:
@ -234,7 +324,7 @@ func (m *Manager) sender() {
select {
case <-m.ctx.Done():
return
case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker.
select {
case <-m.triggerSend:
sentUpdates.WithLabelValues(m.name).Inc()
@ -255,14 +345,18 @@ func (m *Manager) sender() {
}
func (m *Manager) cancelDiscoverers() {
for _, c := range m.discoverCancel {
c()
m.mtx.RLock()
defer m.mtx.RUnlock()
for _, p := range m.providers {
if p.cancel != nil {
p.cancel()
}
}
}
func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) {
m.mtx.Lock()
defer m.mtx.Unlock()
m.targetsMtx.Lock()
defer m.targetsMtx.Unlock()
if _, ok := m.targets[poolKey]; !ok {
m.targets[poolKey] = make(map[string]*targetgroup.Group)
@ -275,11 +369,11 @@ func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) {
}
func (m *Manager) allGroups() map[string][]*targetgroup.Group {
m.mtx.RLock()
defer m.mtx.RUnlock()
tSets := map[string][]*targetgroup.Group{}
n := map[string]int{}
m.targetsMtx.Lock()
defer m.targetsMtx.Unlock()
for pkey, tsets := range m.targets {
for _, tg := range tsets {
// Even if the target group 'tg' is empty we still need to send it to the 'Scrape manager'
@ -303,7 +397,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
add := func(cfg Config) {
for _, p := range m.providers {
if reflect.DeepEqual(cfg, p.config) {
p.subs = append(p.subs, setName)
p.newSubs[setName] = struct{}{}
added = true
return
}
@ -318,11 +412,14 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
return
}
m.providers = append(m.providers, &provider{
name: fmt.Sprintf("%s/%d", typ, len(m.providers)),
name: fmt.Sprintf("%s/%d", typ, m.lastProvider),
d: d,
config: cfg,
subs: []string{setName},
newSubs: map[string]struct{}{
setName: {},
},
})
m.lastProvider++
added = true
}
for _, cfg := range cfgs {

399
discovery/manager_test.go
View file

@ -18,6 +18,7 @@ import (
"fmt"
"sort"
"strconv"
"sync"
"testing"
"time"
@ -36,7 +37,6 @@ func TestMain(m *testing.M) {
// TestTargetUpdatesOrder checks that the target updates are received in the expected order.
func TestTargetUpdatesOrder(t *testing.T) {
// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
// Final targets array is ordered alphabetically by the name of the discoverer.
// For example discoverer "A" with targets "t2,t3" and discoverer "B" with targets "t1,t2" will result in "t2,t3,t1,t2" after the merge.
@ -116,7 +116,8 @@ func TestTargetUpdatesOrder(t *testing.T) {
{
Source: "tp1_group2",
Targets: []model.LabelSet{{"__instance__": "2"}},
}},
},
},
},
},
},
@ -718,6 +719,31 @@ func staticConfig(addrs ...string) StaticConfig {
return cfg
}
func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group, key, label string, present bool) {
t.Helper()
if _, ok := tGroups[key]; !ok {
t.Fatalf("'%s' should be present in Group map keys: %v", key, tGroups)
return
}
match := false
var mergedTargets string
for _, targetGroups := range tGroups[key] {
for _, l := range targetGroups.Targets {
mergedTargets = mergedTargets + " " + l.String()
if l.String() == label {
match = true
}
}
}
if match != present {
msg := ""
if !present {
msg = "not"
}
t.Fatalf("%q should %s be present in Group labels: %q", label, msg, mergedTargets)
}
}
func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
t.Helper()
if _, ok := tSets[poolKey]; !ok {
@ -728,14 +754,12 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
match := false
var mergedTargets string
for _, targetGroup := range tSets[poolKey] {
for _, l := range targetGroup.Targets {
mergedTargets = mergedTargets + " " + l.String()
if l.String() == label {
match = true
}
}
}
if match != present {
msg := ""
@ -746,7 +770,180 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
}
}
func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
func pk(provider, setName string, n int) poolKey {
return poolKey{
setName: setName,
provider: fmt.Sprintf("%s/%d", provider, n),
}
}
func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
discoveryManager.updatert = 100 * time.Millisecond
go discoveryManager.Run()
c := map[string]Configs{
"prometheus": {
staticConfig("foo:9090"),
},
}
discoveryManager.ApplyConfig(c)
syncedTargets := <-discoveryManager.SyncCh()
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
discoveryManager.ApplyConfig(c)
syncedTargets = <-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
}
func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
discoveryManager.updatert = 100 * time.Millisecond
go discoveryManager.Run()
c := map[string]Configs{
"prometheus": {
staticConfig("foo:9090"),
},
}
discoveryManager.ApplyConfig(c)
syncedTargets := <-discoveryManager.SyncCh()
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
c["prometheus2"] = c["prometheus"]
delete(c, "prometheus")
discoveryManager.ApplyConfig(c)
syncedTargets = <-discoveryManager.SyncCh()
p = pk("static", "prometheus2", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus2", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus2"]))
}
func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
discoveryManager.updatert = 100 * time.Millisecond
go discoveryManager.Run()
c := map[string]Configs{
"prometheus": {
staticConfig("foo:9090"),
},
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
c["prometheus2"] = c["prometheus"]
discoveryManager.ApplyConfig(c)
syncedTargets := <-discoveryManager.SyncCh()
require.Equal(t, 2, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
verifySyncedPresence(t, syncedTargets, "prometheus2", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus2"]))
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 2, len(discoveryManager.targets))
delete(c, "prometheus")
discoveryManager.ApplyConfig(c)
syncedTargets = <-discoveryManager.SyncCh()
p = pk("static", "prometheus2", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus2", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus2"]))
}
func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
discoveryManager.updatert = 100 * time.Millisecond
go discoveryManager.Run()
c := map[string]Configs{
"prometheus": {
staticConfig("foo:9090"),
},
}
discoveryManager.ApplyConfig(c)
syncedTargets := <-discoveryManager.SyncCh()
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
var mu sync.Mutex
c["prometheus2"] = Configs{
lockStaticConfig{
mu: &mu,
config: staticConfig("bar:9090"),
},
}
mu.Lock()
discoveryManager.ApplyConfig(c)
// Original targets should be present as soon as possible.
syncedTargets = <-discoveryManager.SyncCh()
mu.Unlock()
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
// prometheus2 configs should be ready on second sync.
syncedTargets = <-discoveryManager.SyncCh()
require.Equal(t, 2, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
verifySyncedPresence(t, syncedTargets, "prometheus2", "{__address__=\"bar:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus2"]))
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
p = pk("lockstatic", "prometheus2", 1)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"bar:9090\"}", true)
require.Equal(t, 2, len(discoveryManager.targets))
// Delete part of config and ensure only original targets exist.
delete(c, "prometheus2")
discoveryManager.ApplyConfig(c)
syncedTargets = <-discoveryManager.SyncCh()
require.Equal(t, 1, len(discoveryManager.targets))
verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
}
func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
@ -760,18 +957,29 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
syncedTargets := <-discoveryManager.SyncCh()
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"bar:9090\"}", true)
require.Equal(t, 1, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"bar:9090\"}", true)
require.Equal(t, 2, len(syncedTargets["prometheus"]))
c["prometheus"] = Configs{
staticConfig("foo:9090"),
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", false)
syncedTargets = <-discoveryManager.SyncCh()
require.Equal(t, 1, len(discoveryManager.targets))
p = pk("static", "prometheus", 1)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"bar:9090\"}", false)
require.Equal(t, 1, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
}
func TestDiscovererConfigs(t *testing.T) {
@ -789,10 +997,18 @@ func TestDiscovererConfigs(t *testing.T) {
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/1"}, "{__address__=\"baz:9090\"}", true)
syncedTargets := <-discoveryManager.SyncCh()
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"bar:9090\"}", true)
p = pk("static", "prometheus", 1)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"baz:9090\"}", true)
require.Equal(t, 2, len(discoveryManager.targets))
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"bar:9090\"}", true)
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"baz:9090\"}", true)
require.Equal(t, 3, len(syncedTargets["prometheus"]))
}
// TestTargetSetRecreatesEmptyStaticConfigs ensures that reloading a config file after
@ -812,20 +1028,23 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
syncedTargets := <-discoveryManager.SyncCh()
p := pk("static", "prometheus", 0)
verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
c["prometheus"] = Configs{
StaticConfig{{}},
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
pkey := poolKey{setName: "prometheus", provider: "static/0"}
targetGroups, ok := discoveryManager.targets[pkey]
syncedTargets = <-discoveryManager.SyncCh()
p = pk("static", "prometheus", 1)
targetGroups, ok := discoveryManager.targets[p]
if !ok {
t.Fatalf("'%v' should be present in target groups", pkey)
t.Fatalf("'%v' should be present in target groups", p)
}
group, ok := targetGroups[""]
if !ok {
@ -835,6 +1054,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
if len(group.Targets) != 0 {
t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
}
require.Equal(t, 1, len(syncedTargets))
require.Equal(t, 1, len(syncedTargets["prometheus"]))
if lbls := syncedTargets["prometheus"][0].Labels; lbls != nil {
t.Fatalf("Unexpected Group: expected nil Labels, got %v", lbls)
}
}
func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
@ -854,12 +1078,17 @@ func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
}
discoveryManager.ApplyConfig(c)
<-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus2", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
syncedTargets := <-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, pk("static", "prometheus2", 0), "{__address__=\"foo:9090\"}", true)
if len(discoveryManager.providers) != 1 {
t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
}
require.Equal(t, 2, len(syncedTargets))
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus"]))
verifySyncedPresence(t, syncedTargets, "prometheus2", "{__address__=\"foo:9090\"}", true)
require.Equal(t, 1, len(syncedTargets["prometheus2"]))
}
func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
@ -891,6 +1120,29 @@ type errorConfig struct{ err error }
func (e errorConfig) Name() string { return "error" }
func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }
type lockStaticConfig struct {
mu *sync.Mutex
config StaticConfig
}
func (s lockStaticConfig) Name() string { return "lockstatic" }
func (s lockStaticConfig) NewDiscoverer(options DiscovererOptions) (Discoverer, error) {
return (lockStaticDiscoverer)(s), nil
}
type lockStaticDiscoverer lockStaticConfig
func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {
// TODO: existing implementation closes up chan, but documentation explicitly forbids it...?
defer close(up)
s.mu.Lock()
defer s.mu.Unlock()
select {
case <-ctx.Done():
case up <- s.config:
}
}
func TestGaugeFailedConfigs(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -923,7 +1175,6 @@ func TestGaugeFailedConfigs(t *testing.T) {
if failedCount != 0 {
t.Fatalf("Expected to get no failed config, got: %v", failedCount)
}
}
func TestCoordinationWithReceiver(t *testing.T) {
@ -1115,7 +1366,11 @@ func (tp mockdiscoveryProvider) Run(ctx context.Context, upCh chan<- []*targetgr
for i := range u.targetGroups {
tgs[i] = &u.targetGroups[i]
}
upCh <- tgs
select {
case <-ctx.Done():
return
case upCh <- tgs:
}
}
<-ctx.Done()
}
@ -1138,3 +1393,91 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
}
close(ch)
}
// TestTargetSetTargetGroupsUpdateDuringApplyConfig is used to detect races when
// ApplyConfig happens at the same time as targets update.
func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveryManager := NewManager(ctx, log.NewNopLogger())
discoveryManager.updatert = 100 * time.Millisecond
go discoveryManager.Run()
td := newTestDiscoverer()
c := map[string]Configs{
"prometheus": {
td,
},
}
discoveryManager.ApplyConfig(c)
var wg sync.WaitGroup
wg.Add(2000)
start := make(chan struct{})
for i := 0; i < 1000; i++ {
go func() {
<-start
td.update([]*targetgroup.Group{
{
Targets: []model.LabelSet{
{model.AddressLabel: model.LabelValue("127.0.0.1:9090")},
},
},
})
wg.Done()
}()
}
for i := 0; i < 1000; i++ {
go func(i int) {
<-start
c := map[string]Configs{
fmt.Sprintf("prometheus-%d", i): {
td,
},
}
discoveryManager.ApplyConfig(c)
wg.Done()
}(i)
}
close(start)
wg.Wait()
}
// testDiscoverer is a config and a discoverer that can adjust targets with a
// simple function.
type testDiscoverer struct {
up chan<- []*targetgroup.Group
ready chan struct{}
}
func newTestDiscoverer() *testDiscoverer {
return &testDiscoverer{
ready: make(chan struct{}),
}
}
// Name implements Config.
func (t *testDiscoverer) Name() string {
return "test"
}
// NewDiscoverer implements Config.
func (t *testDiscoverer) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
return t, nil
}
// Run implements Discoverer.
func (t *testDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {
t.up = up
close(t.ready)
<-ctx.Done()
}
func (t *testDiscoverer) update(tgs []*targetgroup.Group) {
<-t.ready
t.up <- tgs
}

2
discovery/marathon/marathon.go
View file

@ -478,7 +478,6 @@ func targetsForApp(app *app) []model.LabelSet {
// Generate a target endpoint string in host:port format.
func targetEndpoint(task *task, port uint32, containerNet bool) string {
var host string
// Use the task's ipAddress field when it's in a container network
@ -493,7 +492,6 @@ func targetEndpoint(task *task, port uint32, containerNet bool) string {
// Get a list of ports and a list of labels from a PortMapping.
func extractPortMapping(portMappings []portMapping, containerNet bool) ([]uint32, []map[string]string) {
ports := make([]uint32, len(portMappings))
labels := make([]map[string]string, len(portMappings))

92
discovery/marathon/marathon_test.go
View file

@ -29,11 +29,14 @@ import (
var (
marathonValidLabel = map[string]string{"prometheus": "yes"}
testServers = []string{"http://localhost:8080"}
conf = SDConfig{Servers: testServers}
)
func testConfig() SDConfig {
return SDConfig{Servers: testServers}
}
func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
md, err := NewDiscovery(conf, nil)
md, err := NewDiscovery(testConfig(), nil)
if err != nil {
return nil, err
}
@ -60,9 +63,7 @@ func TestMarathonSDHandleError(t *testing.T) {
}
func TestMarathonSDEmptyList(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -99,11 +100,9 @@ func marathonTestAppList(labels map[string]string, runningTasks int) *appList {
}
func TestMarathonSDSendGroup(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppList(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppList(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -130,7 +129,7 @@ func TestMarathonSDSendGroup(t *testing.T) {
}
func TestMarathonSDRemoveApp(t *testing.T) {
md, err := NewDiscovery(conf, nil)
md, err := NewDiscovery(testConfig(), nil)
if err != nil {
t.Fatalf("%s", err)
}
@ -195,11 +194,9 @@ func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks
}
func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -254,11 +251,9 @@ func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int)
}
func TestMarathonZeroTaskPorts(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -286,13 +281,6 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
// Create a test server with mock HTTP handler.
ts := httptest.NewServer(http.HandlerFunc(respHandler))
defer ts.Close()
// Backup conf for future tests.
backupConf := conf
defer func() {
conf = backupConf
}()
// Setup conf for the test case.
conf = SDConfig{Servers: []string{ts.URL}}
// Execute test case and validate behavior.
_, err := testUpdateServices(nil)
if err == nil {
@ -331,11 +319,9 @@ func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTas
}
func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -403,11 +389,9 @@ func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string
}
func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -470,11 +454,9 @@ func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *a
}
func TestMarathonSDSendGroupWithPorts(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -546,11 +528,9 @@ func marathonTestAppListWithContainerPortMappings(labels map[string]string, runn
}
func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -622,11 +602,9 @@ func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string
}
func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
@ -702,11 +680,9 @@ func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]st
}
func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
var (
client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
}
)
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)

1
discovery/moby/docker.go
View file

@ -28,6 +28,7 @@ import (
"github.com/go-kit/log"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"

1
discovery/moby/network.go
View file

@ -19,6 +19,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/prometheus/prometheus/util/strutil"
)

6
discovery/openstack/hypervisor.go
View file

@ -51,8 +51,10 @@ type HypervisorDiscovery struct {
// newHypervisorDiscovery returns a new hypervisor discovery.
func newHypervisorDiscovery(provider *gophercloud.ProviderClient, opts *gophercloud.AuthOptions,
port int, region string, availability gophercloud.Availability, l log.Logger) *HypervisorDiscovery {
return &HypervisorDiscovery{provider: provider, authOpts: opts,
region: region, port: port, availability: availability, logger: l}
return &HypervisorDiscovery{
provider: provider, authOpts: opts,
region: region, port: port, availability: availability, logger: l,
}
}
func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {

1
discovery/openstack/hypervisor_test.go
View file

@ -47,7 +47,6 @@ func (s *OpenstackSDHypervisorTestSuite) openstackAuthSuccess() (refresher, erro
}
func TestOpenstackSDHypervisorRefresh(t *testing.T) {
mock := &OpenstackSDHypervisorTestSuite{}
mock.SetupTest(t)

6
discovery/openstack/instance.go
View file

@ -63,8 +63,10 @@ func newInstanceDiscovery(provider *gophercloud.ProviderClient, opts *gopherclou
if l == nil {
l = log.NewNopLogger()
}
return &InstanceDiscovery{provider: provider, authOpts: opts,
region: region, port: port, allTenants: allTenants, availability: availability, logger: l}
return &InstanceDiscovery{
provider: provider, authOpts: opts,
region: region, port: port, allTenants: allTenants, availability: availability, logger: l,
}
}
type floatingIPKey struct {

1
discovery/openstack/instance_test.go
View file

@ -51,7 +51,6 @@ func (s *OpenstackSDInstanceTestSuite) openstackAuthSuccess() (refresher, error)
}
func TestOpenstackSDInstanceRefresh(t *testing.T) {
mock := &OpenstackSDInstanceTestSuite{}
mock.SetupTest(t)

2
discovery/openstack/mock_test.go
View file

@ -54,7 +54,7 @@ func testMethod(t *testing.T, r *http.Request, expected string) {
}
}
func testHeader(t *testing.T, r *http.Request, header string, expected string) {
func testHeader(t *testing.T, r *http.Request, header, expected string) {
if actual := r.Header.Get(header); expected != actual {
t.Errorf("Header %s = %s, expected %s", header, actual, expected)
}

1
discovery/openstack/openstack.go
View file

@ -145,7 +145,6 @@ func NewDiscovery(conf *SDConfig, l log.Logger) (*refresh.Discovery, error) {
time.Duration(conf.RefreshInterval),
r.refresh,
), nil
}
func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) {

3
discovery/puppetdb/puppetdb_test.go
View file

@ -25,8 +25,9 @@ import (
"github.com/go-kit/log"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func mockServer(t *testing.T) *httptest.Server {

1
discovery/puppetdb/resources.go
View file

@ -18,6 +18,7 @@ import (
"strings"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/util/strutil"
)

5
discovery/scaleway/baremetal.go
View file

@ -25,10 +25,11 @@ import (
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/scaleway/scaleway-sdk-go/api/baremetal/v1"
"github.com/scaleway/scaleway-sdk-go/scw"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
type baremetalDiscovery struct {

5
discovery/scaleway/instance.go
View file

@ -25,10 +25,11 @@ import (
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
"github.com/scaleway/scaleway-sdk-go/scw"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
const (

6
discovery/scaleway/scaleway.go
View file

@ -24,10 +24,11 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/scaleway/scaleway-sdk-go/scw"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/scaleway/scaleway-sdk-go/scw"
)
// metaLabelPrefix is the meta prefix used for all meta labels.
@ -173,8 +174,7 @@ func init() {
// Discovery periodically performs Scaleway requests. It implements
// the Discoverer interface.
type Discovery struct {
}
type Discovery struct{}
func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
r, err := newRefresher(conf)

38
discovery/targetgroup/targetgroup_test.go
View file

@ -38,7 +38,8 @@ func TestTargetGroupStrictJsonUnmarshal(t *testing.T) {
expectedReply: nil,
expectedGroup: Group{Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9091"}}, Labels: model.LabelSet{"my": "label"}},
{"__address__": "localhost:9091"},
}, Labels: model.LabelSet{"my": "label"}},
},
{
json: ` {"label": {},"targets": []}`,
@ -56,7 +57,6 @@ func TestTargetGroupStrictJsonUnmarshal(t *testing.T) {
require.Equal(t, test.expectedReply, actual)
require.Equal(t, test.expectedGroup, tg)
}
}
func TestTargetGroupYamlMarshal(t *testing.T) {
@ -81,10 +81,13 @@ func TestTargetGroupYamlMarshal(t *testing.T) {
},
{
// targets only exposes addresses.
group: Group{Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9091"}},
Labels: model.LabelSet{"foo": "bar", "bar": "baz"}},
group: Group{
Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9091"},
},
Labels: model.LabelSet{"foo": "bar", "bar": "baz"},
},
expectedYaml: "targets:\n- localhost:9090\n- localhost:9091\nlabels:\n bar: baz\n foo: bar\n",
expectedErr: nil,
},
@ -120,7 +123,8 @@ func TestTargetGroupYamlUnmarshal(t *testing.T) {
expectedReply: nil,
expectedGroup: Group{Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9191"}}, Labels: model.LabelSet{"my": "label"}},
{"__address__": "localhost:9191"},
}, Labels: model.LabelSet{"my": "label"}},
},
{
// incorrect syntax.
@ -135,21 +139,25 @@ func TestTargetGroupYamlUnmarshal(t *testing.T) {
require.Equal(t, test.expectedReply, actual)
require.Equal(t, test.expectedGroup, tg)
}
}
func TestString(t *testing.T) {
// String() should return only the source, regardless of other attributes.
group1 :=
Group{Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9091"}},
Group{
Targets: []model.LabelSet{
{"__address__": "localhost:9090"},
{"__address__": "localhost:9091"},
},
Source: "<source>",
Labels: model.LabelSet{"foo": "bar", "bar": "baz"}}
Labels: model.LabelSet{"foo": "bar", "bar": "baz"},
}
group2 :=
Group{Targets: []model.LabelSet{},
Source: "<source>",
Labels: model.LabelSet{}}
Group{
Targets: []model.LabelSet{},
Source: "<source>",
Labels: model.LabelSet{},
}
require.Equal(t, "<source>", group1.String())
require.Equal(t, "<source>", group2.String())
require.Equal(t, group1.String(), group2.String())

6
discovery/triton/triton.go
View file

@ -188,9 +188,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
case "cn":
endpointFormat = "https://%s:%d/v%d/gz/discover"
default:
return nil, errors.New(fmt.Sprintf("unknown role '%s' in configuration", d.sdConfig.Role))
return nil, fmt.Errorf("unknown role '%s' in configuration", d.sdConfig.Role)
}
var endpoint = fmt.Sprintf(endpointFormat, d.sdConfig.Endpoint, d.sdConfig.Port, d.sdConfig.Version)
endpoint := fmt.Sprintf(endpointFormat, d.sdConfig.Endpoint, d.sdConfig.Port, d.sdConfig.Version)
if len(d.sdConfig.Groups) > 0 {
groups := url.QueryEscape(strings.Join(d.sdConfig.Groups, ","))
endpoint = fmt.Sprintf("%s?groups=%s", endpoint, groups)
@ -223,7 +223,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
case "cn":
return d.processComputeNodeResponse(data, endpoint)
default:
return nil, errors.New(fmt.Sprintf("unknown role '%s' in configuration", d.sdConfig.Role))
return nil, fmt.Errorf("unknown role '%s' in configuration", d.sdConfig.Role)
}
}

20
discovery/triton/triton_test.go
View file

@ -135,8 +135,7 @@ func TestTritonSDRefreshNoTargets(t *testing.T) {
}
func TestTritonSDRefreshMultipleTargets(t *testing.T) {
var (
dstr = `{"containers":[
dstr := `{"containers":[
{
"groups":["foo","bar","baz"],
"server_uuid":"44454c4c-5000-104d-8037-b7c04f5a5131",
@ -153,7 +152,6 @@ func TestTritonSDRefreshMultipleTargets(t *testing.T) {
"vm_uuid":"7b27a514-89d7-11e6-bee6-3f96f367bee7"
}]
}`
)
tgts := testTritonSDRefresh(t, conf, dstr)
require.NotNil(t, tgts)
@ -161,9 +159,7 @@ func TestTritonSDRefreshMultipleTargets(t *testing.T) {
}
func TestTritonSDRefreshNoServer(t *testing.T) {
var (
td, _ = newTritonDiscovery(conf)
)
td, _ := newTritonDiscovery(conf)
_, err := td.refresh(context.Background())
require.Error(t, err)
@ -171,9 +167,7 @@ func TestTritonSDRefreshNoServer(t *testing.T) {
}
func TestTritonSDRefreshCancelled(t *testing.T) {
var (
td, _ = newTritonDiscovery(conf)
)
td, _ := newTritonDiscovery(conf)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@ -183,8 +177,7 @@ func TestTritonSDRefreshCancelled(t *testing.T) {
}
func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) {
var (
dstr = `{"cns":[
dstr := `{"cns":[
{
"server_uuid":"44454c4c-5000-104d-8037-b7c04f5a5131"
},
@ -192,7 +185,6 @@ func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) {
"server_uuid":"a5894692-bd32-4ca1-908a-e2dda3c3a5e6"
}]
}`
)
tgts := testTritonSDRefresh(t, cnconf, dstr)
require.NotNil(t, tgts)
@ -200,8 +192,7 @@ func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) {
}
func TestTritonSDRefreshCNsWithHostname(t *testing.T) {
var (
dstr = `{"cns":[
dstr := `{"cns":[
{
"server_uuid":"44454c4c-5000-104d-8037-b7c04f5a5131",
"server_hostname": "server01"
@ -211,7 +202,6 @@ func TestTritonSDRefreshCNsWithHostname(t *testing.T) {
"server_hostname": "server02"
}]
}`
)
tgts := testTritonSDRefresh(t, cnconf, dstr)
require.NotNil(t, tgts)

15
discovery/uyuni/uyuni.go
View file

@ -71,7 +71,6 @@ type SDConfig struct {
RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
}
// Uyuni API Response structures
type systemGroupID struct {
GroupID int `xmlrpc:"id"`
GroupName string `xmlrpc:"name"`
@ -120,7 +119,6 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultSDConfig
type plain SDConfig
err := unmarshal((*plain)(c))
if err != nil {
return err
}
@ -142,20 +140,17 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}
// Attempt to login in Uyuni Server and get an auth token
func login(rpcclient *xmlrpc.Client, user string, pass string) (string, error) {
func login(rpcclient *xmlrpc.Client, user, pass string) (string, error) {
var result string
err := rpcclient.Call("auth.login", []interface{}{user, pass}, &result)
return result, err
}
// Logout from Uyuni API
func logout(rpcclient *xmlrpc.Client, token string) error {
return rpcclient.Call("auth.logout", token, nil)
}
// Get the system groups information of monitored clients
func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token string, entitlement string) (map[int][]systemGroupID, error) {
func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token, entitlement string) (map[int][]systemGroupID, error) {
var systemGroupsInfos []struct {
SystemID int `xmlrpc:"id"`
SystemGroups []systemGroupID `xmlrpc:"system_groups"`
@ -173,7 +168,6 @@ func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token strin
return result, nil
}
// GetSystemNetworkInfo lists client FQDNs.
func getNetworkInformationForSystems(rpcclient *xmlrpc.Client, token string, systemIDs []int) (map[int]networkInfo, error) {
var networkInfos []networkInfo
err := rpcclient.Call("system.getNetworkForSystems", []interface{}{token, systemIDs}, &networkInfos)
@ -188,7 +182,6 @@ func getNetworkInformationForSystems(rpcclient *xmlrpc.Client, token string, sys
return result, nil
}
// Get endpoints information for given systems
func getEndpointInfoForSystems(
rpcclient *xmlrpc.Client,
token string,
@ -210,7 +203,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
*apiURL = *conf.Server.URL
apiURL.Path = path.Join(apiURL.Path, uyuniXMLRPCAPIPath)
rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "uyuni_sd", config.WithHTTP2Disabled())
rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "uyuni_sd")
if err != nil {
return nil, err
}
@ -240,7 +233,6 @@ func (d *Discovery) getEndpointLabels(
systemGroupIDs []systemGroupID,
networkInfo networkInfo,
) model.LabelSet {
var addr, scheme string
managedGroupNames := getSystemGroupNames(systemGroupIDs)
addr = fmt.Sprintf("%s:%d", networkInfo.Hostname, endpoint.Port)
@ -280,7 +272,6 @@ func (d *Discovery) getTargetsForSystems(
token string,
entitlement string,
) ([]model.LabelSet, error) {
result := make([]model.LabelSet, 0)
systemGroupIDsBySystemID, err := getSystemGroupsInfoOfMonitoredClients(rpcClient, token, entitlement)

14
discovery/xds/client_test.go
View file

@ -26,8 +26,8 @@ import (
"google.golang.org/protobuf/types/known/anypb"
)
var (
httpResourceConf = &HTTPResourceClientConfig{
func testHTTPResourceConfig() *HTTPResourceClientConfig {
return &HTTPResourceClientConfig{
HTTPClientConfig: config.HTTPClientConfig{
TLSConfig: config.TLSConfig{InsecureSkipVerify: true},
},
@ -37,11 +37,10 @@ var (
Server: "http://localhost",
ClientID: "test-id",
}
)
}
func urlMustParse(str string) *url.URL {
parsed, err := url.Parse(str)
if err != nil {
panic(err)
}
@ -92,7 +91,6 @@ func TestCreateNewHTTPResourceClient(t *testing.T) {
require.Equal(t, client.endpoint, "http://127.0.0.1:5000/v3/discovery:monitoring?param1=v1")
require.Equal(t, client.client.Timeout, 1*time.Minute)
}
func createTestHTTPResourceClient(t *testing.T, conf *HTTPResourceClientConfig, protocolVersion ProtocolVersion, responder discoveryResponder) (*HTTPResourceClient, func()) {
@ -110,7 +108,7 @@ func createTestHTTPResourceClient(t *testing.T, conf *HTTPResourceClientConfig,
}
func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) {
client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
return nil, nil
})
defer cleanup()
@ -121,7 +119,7 @@ func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) {
}
func TestHTTPResourceClientFetchFullResponse(t *testing.T) {
client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
if request.VersionInfo == "1" {
return nil, nil
}
@ -150,7 +148,7 @@ func TestHTTPResourceClientFetchFullResponse(t *testing.T) {
}
func TestHTTPResourceClientServerError(t *testing.T) {
client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) {
return nil, errors.New("server error")
})
defer cleanup()

6
discovery/xds/kuma.go
View file

@ -103,11 +103,7 @@ func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
return errors.Errorf("kuma SD server must not be empty and have a scheme: %s", c.Server)
}
if err := c.HTTPClientConfig.Validate(); err != nil {
return err
}
return nil
return c.HTTPClientConfig.Validate()
}
func (c *KumaSDConfig) Name() string {

1
discovery/xds/kuma_test.go
View file

@ -91,7 +91,6 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis
serialized := make([]*anypb.Any, len(resources))
for i, res := range resources {
data, err := proto.Marshal(res)
if err != nil {
return nil, err
}

44
docs/configuration/configuration.md
View file

@ -383,6 +383,10 @@ token_url: <string>
# Optional parameters to append to the token URL.
endpoint_params:
[ <string>: <string> ... ]
# Configures the token request's TLS settings.
tls_config:
[ <tls_config> ]
```
### `<azure_sd_config>`
@ -429,6 +433,42 @@ subscription_id: <string>
# The port to scrape metrics from. If using the public IP address, this must
# instead be specified in the relabeling rule.
[ port: <int> | default = 80 ]
# Authentication information used to authenticate to the consul server.
# Note that `basic_auth`, `authorization` and `oauth2` options are
# mutually exclusive.
# `password` and `password_file` are mutually exclusive.
# Optional HTTP basic authentication information, currently not support by Azure.
basic_auth:
[ username: <string> ]
[ password: <secret> ]
[ password_file: <string> ]
# Optional `Authorization` header configuration, currently not supported by Azure.
authorization:
# Sets the authentication type.
[ type: <string> | default: Bearer ]
# Sets the credentials. It is mutually exclusive with
# `credentials_file`.
[ credentials: <secret> ]
# Sets the credentials to the credentials read from the configured file.
# It is mutually exclusive with `credentials`.
[ credentials_file: <filename> ]
# Optional OAuth 2.0 configuration, currently not supported by Azure.
oauth2:
[ <oauth2> ]
# Optional proxy URL.
[ proxy_url: <string> ]
# Configure whether HTTP requests follow HTTP 3xx redirects.
[ follow_redirects: <bool> | default = true ]
# TLS configuration.
tls_config:
[ <tls_config> ]
```
### `<consul_sd_config>`
@ -1563,7 +1603,7 @@ Available meta labels:
from underlying pods), the following labels are attached:
* `__meta_kubernetes_endpointslice_address_target_kind`: Kind of the referenced object.
* `__meta_kubernetes_endpointslice_address_target_name`: Name of referenced object.
* `__meta_kubernetes_endpointslice_address_type`: The ip protocol family of the adress target.
* `__meta_kubernetes_endpointslice_address_type`: The ip protocol family of the address of the target.
* `__meta_kubernetes_endpointslice_endpoint_conditions_ready`: Set to `true` or `false` for the referenced endpoint's ready state.
* `__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname`: Name of the node hosting the referenced endpoint.
* `__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname`: Flag that shows if the referenced object has a kubernetes.io/hostname annotation.
@ -2712,7 +2752,7 @@ queue_config:
# Initial retry delay. Gets doubled for every retry.
[ min_backoff: <duration> | default = 30ms ]
# Maximum retry delay.
[ max_backoff: <duration> | default = 100ms ]
[ max_backoff: <duration> | default = 5s ]
# Retry upon receiving a 429 status code from the remote-write storage.
# This is experimental and might change in the future.
[ retry_on_http_429: <boolean> | default = false ]

24
docs/configuration/https.md
View file

@ -73,6 +73,30 @@ http_server_config:
# Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS.
# This can not be changed on the fly.
[ http2: <boolean> | default = true ]
# List of headers that can be added to HTTP responses.
[ headers:
# Set the Content-Security-Policy header to HTTP responses.
# Unset if blank.
[ Content-Security-Policy: <string> ]
# Set the X-Frame-Options header to HTTP responses.
# Unset if blank. Accepted values are deny and sameorigin.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
[ X-Frame-Options: <string> ]
# Set the X-Content-Type-Options header to HTTP responses.
# Unset if blank. Accepted value is nosniff.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options
[ X-Content-Type-Options: <string> ]
# Set the X-XSS-Protection header to all responses.
# Unset if blank. Accepted value is nosniff.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection
[ X-XSS-Protection: <string> ]
# Set the Strict-Transport-Security header to HTTP responses.
# Unset if blank.
# Please make sure that you use this with care as this header might force
# browsers to load Prometheus and the other applications hosted on the same
# domain and subdomains over HTTPS.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security
[ Strict-Transport-Security: <string> ] ]
# Usernames and hashed passwords that have full access to the web
# server via basic authentication. If empty, no basic authentication is

12
docs/configuration/recording_rules.md
View file

@ -78,8 +78,8 @@ name: <string>
# How often rules in the group are evaluated.
[ interval: <duration> | default = global.evaluation_interval ]
# Limit the number of alerts and series individual rules can produce.
# 0 is no limit.
# Limit the number of alerts an alerting rule and series a recording
# rule can produce. 0 is no limit.
[ limit: <int> | default = 0 ]
rules:
@ -128,3 +128,11 @@ annotations:
[ <labelname>: <tmpl_string> ]
```
# Limiting alerts and series
A limit for alerts produced by alerting rules and series produced recording rules
can be configured per-group. When the limit is exceeded, _all_ series produced
by the rule are discarded, and if it's an alerting rule, _all_ alerts for
the rule, active, pending, or inactive, are cleared as well. The event will be
recorded as an error in the evaluation, and as such no stale markers are
written.

1
docs/configuration/template_reference.md
View file

@ -74,6 +74,7 @@ versions.
| reReplaceAll | pattern, replacement, text | string | [Regexp.ReplaceAllString](https://golang.org/pkg/regexp/#Regexp.ReplaceAllString) Regexp substitution, unanchored. |
| graphLink | expr | string | Returns path to graph view in the [expression browser](https://prometheus.io/docs/visualization/browser/) for the expression. |
| tableLink | expr | string | Returns path to tabular ("Table") view in the [expression browser](https://prometheus.io/docs/visualization/browser/) for the expression. |
| parseDuration | string | float | Parses a duration string such as "1h" into the number of seconds it represents. |
### Others

35
docs/feature_flags.md
View file

@ -1,9 +1,9 @@
---
title: Feature Flags
title: Feature flags
sort_rank: 11
---
# Feature Flags
# Feature flags
Here is a list of features that are disabled by default since they are breaking changes or are considered experimental.
Their behaviour can change in future releases which will be communicated via the [release changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md).
@ -46,7 +46,7 @@ More details can be found [here](querying/basics.md#offset-modifier).
The remote write receiver allows Prometheus to accept remote write requests from other Prometheus servers. More details can be found [here](storage.md#overview).
## Exemplars Storage
## Exemplars storage
`--enable-feature=exemplar-storage`
@ -54,7 +54,7 @@ The remote write receiver allows Prometheus to accept remote write requests from
Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The flag `storage.exemplars.exemplars-limit` can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `traceID=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
## Memory Snapshot on Shutdown
## Memory snapshot on shutdown
`--enable-feature=memory-snapshot-on-shutdown`
@ -62,7 +62,7 @@ This takes the snapshot of the chunks that are in memory along with the series i
it on disk. This will reduce the startup time since the memory state can be restored with this snapshot and m-mapped
chunks without the need of WAL replay.
## Extra Scrape Metrics
## Extra scrape metrics
`--enable-feature=extra-scrape-metrics`
@ -71,3 +71,28 @@ When enabled, for each instance scrape, Prometheus stores a sample in the follow
- `scrape_timeout_seconds`. The configured `scrape_timeout` for a target. This allows you to measure each target to find out how close they are to timing out with `scrape_duration_seconds / scrape_timeout_seconds`.
- `scrape_sample_limit`. The configured `sample_limit` for a target. This allows you to measure each target
to find out how close they are to reaching the limit with `scrape_samples_post_metric_relabeling / scrape_sample_limit`. Note that `scrape_sample_limit` can be zero if there is no limit configured, which means that the query above can return `+Inf` for targets with no limit (as we divide by zero). If you want to query only for targets that do have a sample limit use this query: `scrape_samples_post_metric_relabeling / (scrape_sample_limit > 0)`.
- `scrape_body_size_bytes`. The uncompressed size of the most recent scrape response, if successful. Scrapes failing because `body_size_limit` is exceeded report `-1`, other scrape failures report `0`.
## New service discovery manager
`--enable-feature=new-service-discovery-manager`
When enabled, Prometheus uses a new service discovery manager that does not
restart unchanged discoveries upon reloading. This makes reloads faster and reduces
pressure on service discoveries' sources.
Users are encouraged to test the new service discovery manager and report any
issues upstream.
In future releases, this new service discovery manager will become the default and
this feature flag will be ignored.
## Prometheus agent
`--enable-feature=agent`
When enabled, Prometheus runs in agent mode. The agent mode is limited to
discovery, scrape and remote write.
This is useful when you do not need to query the Prometheus data locally, but
only from a central [remote endpoint](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).

2
docs/querying/operators.md
View file

@ -222,7 +222,7 @@ or
both `(label1, label2)` and `(label1, label2,)` are valid syntax.
`without` removes the listed labels from the result vector, while
all other labels are preserved the output. `by` does the opposite and drops
all other labels are preserved in the output. `by` does the opposite and drops
labels that are not listed in the `by` clause, even if their label values are
identical between all elements of the vector.

5
documentation/examples/custom-sd/adapter-usage/main.go
View file

@ -50,7 +50,7 @@ var (
tagsLabel = model.MetaLabelPrefix + "consul_tags"
// serviceAddressLabel is the name of the label containing the (optional) service address.
serviceAddressLabel = model.MetaLabelPrefix + "consul_service_address"
//servicePortLabel is the name of the label containing the service port.
// servicePortLabel is the name of the label containing the service port.
servicePortLabel = model.MetaLabelPrefix + "consul_service_port"
// serviceIDLabel is the name of the label containing the service ID.
serviceIDLabel = model.MetaLabelPrefix + "consul_service_id"
@ -120,7 +120,7 @@ func (d *discovery) parseServiceNodes(resp *http.Response, name string) (*target
for _, node := range nodes {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
var tags = "," + strings.Join(node.ServiceTags, ",") + ","
tags := "," + strings.Join(node.ServiceTags, ",") + ","
// If the service address is not empty it should be used instead of the node address
// since the service may be registered remotely through a different node.
@ -162,7 +162,6 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
for c := time.Tick(time.Duration(d.refreshInterval) * time.Second); ; {
var srvs map[string][]string
resp, err := http.Get(fmt.Sprintf("http://%s/v1/catalog/services", d.address))
if err != nil {
level.Error(d.logger).Log("msg", "Error getting services list", "err", err)
time.Sleep(time.Duration(d.refreshInterval) * time.Second)

2
documentation/examples/custom-sd/adapter/adapter.go
View file

@ -163,7 +163,7 @@ func (a *Adapter) Run() {
}
// NewAdapter creates a new instance of Adapter.
func NewAdapter(ctx context.Context, file string, name string, d discovery.Discoverer, logger log.Logger) *Adapter {
func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger) *Adapter {
return &Adapter{
ctx: ctx,
disc: d,

22
documentation/examples/prometheus-agent.yml Normal file
View file

@ -0,0 +1,22 @@
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: "prometheus"
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ["localhost:9090"]
# When running prometheus in Agent mode, remote-write is required.
remote_write:
# Agent is able to run with a invalid remote-write URL, but, of course, will fail to push timeseries.
- url: "http://remote-write-url"

9
documentation/examples/prometheus-puppetdb.yml
View file

@ -14,11 +14,10 @@ scrape_configs:
- job_name: 'puppetdb-scrape-jobs'
puppetdb_sd_configs:
# This example uses the Prometheus::Scrape_job
# exported resources.
# https://github.com/camptocamp/prometheus-puppetdb-sd
# This examples is compatible with Prometheus-puppetdb-sd,
# if the exported Prometheus::Scrape_job only have at most one target.
# This example uses Prometheus::Scrape_job exported resources.
# It is compatible with the prometheus-puppetdb-sd
# (https://github.com/camptocamp/prometheus-puppetdb-sd) if the
# exported resources have exactly one target.
- url: https://puppetdb.example.com
query: 'resources { type = "Prometheus::Scrape_job" and exported = true }'
include_parameters: true

2
documentation/examples/remote_storage/remote_storage_adapter/graphite/client.go
View file

@ -37,7 +37,7 @@ type Client struct {
}
// NewClient creates a new Client.
func NewClient(logger log.Logger, address string, transport string, timeout time.Duration, prefix string) *Client {
func NewClient(logger log.Logger, address, transport string, timeout time.Duration, prefix string) *Client {
if logger == nil {
logger = log.NewNopLogger()
}

12
documentation/examples/remote_storage/remote_storage_adapter/graphite/client_test.go
View file

@ -20,13 +20,11 @@ import (
"github.com/stretchr/testify/require"
)
var (
metric = model.Metric{
model.MetricNameLabel: "test:metric",
"testlabel": "test:value",
"many_chars": "abc!ABC:012-3!45ö67~89./(){},=.\"\\",
}
)
var metric = model.Metric{
model.MetricNameLabel: "test:metric",
"testlabel": "test:value",
"many_chars": "abc!ABC:012-3!45ö67~89./(){},=.\"\\",
}
func TestEscape(t *testing.T) {
// Can we correctly keep and escape valid chars.

2
documentation/examples/remote_storage/remote_storage_adapter/influxdb/client.go
View file

@ -41,7 +41,7 @@ type Client struct {
}
// NewClient creates a new Client.
func NewClient(logger log.Logger, conf influx.HTTPConfig, db string, rp string) *Client {
func NewClient(logger log.Logger, conf influx.HTTPConfig, db, rp string) *Client {
c, err := influx.NewHTTPClient(conf)
// Currently influx.NewClient() *should* never return an error.
if err != nil {

12
documentation/examples/remote_storage/remote_storage_adapter/opentsdb/client_test.go
View file

@ -21,13 +21,11 @@ import (
"github.com/stretchr/testify/require"
)
var (
metric = model.Metric{
model.MetricNameLabel: "test:metric",
"testlabel": "test:value",
"many_chars": "abc!ABC:012-3!45ö67~89./",
}
)
var metric = model.Metric{
model.MetricNameLabel: "test:metric",
"testlabel": "test:value",
"many_chars": "abc!ABC:012-3!45ö67~89./",
}
func TestTagsFromMetric(t *testing.T) {
expected := map[string]TagValue{

4
documentation/prometheus-mixin/Makefile
View file

@ -21,5 +21,9 @@ lint: prometheus_alerts.yaml
promtool check rules prometheus_alerts.yaml
.PHONY: jb_install
jb_install:
jb install
clean:
rm -rf dashboards_out prometheus_alerts.yaml

8
documentation/prometheus-mixin/go.mod
View file

@ -1,8 +0,0 @@
module github.com/prometheus/prometheus/documentation/prometheus-mixin
go 1.15
require (
github.com/google/go-jsonnet v0.16.0
github.com/jsonnet-bundler/jsonnet-bundler v0.4.0
)

Some files were not shown because too many files have changed in this diff Show more